def run(self):
self.data = feedparser.parse(self.feed['url'])
# Update title if it change (yes some guys change it...)
if self.data.feed['title'] != self.feed['title']:
self.feed['title'] = self.data.feed['title']
storage.update_feed(self.feed['_id'], copy.copy(self.feed))
local_update = get_datetime(self.feed['last_update'])
remote_update = False
if self.data.feed.get('updated_parsed'):
remote_update = get_datetime(self.data.feed.updated_parsed)
remote_update_raw = get_dicttime(self.data.feed.updated_parsed)
if self.data.get('updated_parsed'):
if remote_update:
if get_datetime(self.data.updated_parsed) > remote_update:
remote_update = get_datetime(self.data.updated_parsed)
remote_update_raw = get_dicttime(self.data.updated_parsed)
if self.data.feed.get('published_parsed'):
if remote_update:
if get_datetime(self.data.feed.published_parsed) > remote_update:
remote_update = get_datetime(self.data.feed.published_parsed)
remote_update_raw = get_dicttime(self.data.feed.published_parsed)
if self.data.get('published_parsed'):
if remote_update:
if get_datetime(self.data.published_parsed > remote_update):
remote_update = get_datetime(self.data.published_parsed)
remote_update_raw = get_dicttime(self.data.published_parsed)
if not remote_update:
remote_update = datetime.datetime.now()
remote_update_raw = get_dicttime(remote_update.timetuple())
if remote_update > local_update:
print('!! %s is outdated.' % self.feed['title'].encode('utf-8'))
readed = []
for entry in storage.get_stories(self.feed['_id']):
if entry['read']:
readed.append(entry['title'])
if len(self.data.entries) <= int(config.get('worker', 'story_before_retention')):
do_retention = False
else:
do_retention = True
retriever = Retriever(self.data, self.feed, do_retention=do_retention)
retriever.start()
retriever.join()
for entry in readed:
if storage.get_story_by_title(entry):
entry = storage.get_story_by_title(entry)
entry['read'] = True
storage.update_story(entry['_id'], copy.copy(entry))
self.feed['last_update'] = remote_update_raw
storage.update_feed(self.feed_id, self.feed)
else:
print('=> %s is up-to-date.' % self.feed['title'].encode('utf-8'))
def run(self):
# This feed comes from database
feed = backend.get_feed_by_title(self.title)
url = feed['url']
for entry in self.data:
title = entry['title']
link = entry['link']
try:
description = entry['content'][0]['value']
except KeyError:
description = entry['summary']
last_update = get_dicttime(entry.updated_parsed)
if entry.get('published_parsed', False):
published = get_dicttime(entry.published_parsed)
else:
published = None
_id = backend.add_story({
'title': title,
'link': link,
'description': description,
'published': published,
'last_update': last_update,
'feed_id': feed['_id'],
'read': False})
def run(self):
# This feed comes from database
feed = backend.get_feed_by_title(self.title)
url = feed['url']
for entry in self.data:
title = entry['title']
link = entry['link']
try:
description = entry['content'][0]['value']
except KeyError:
description = entry['summary']
last_update = get_dicttime(entry.updated_parsed)
if entry.get('published_parsed', False):
published = get_dicttime(entry.published_parsed)
else:
published = None
_id = backend.add_story({
'title': title,
'link': link,
'description': description,
'published': published,
'last_update': last_update,
'feed_id': feed['_id'],
'read': False
})
def run(self):
# This feed comes from database
feed = storage.get_feed_by_title(self.title)
for entry in self.data:
title = entry['title']
link = entry['link']
try:
description = entry['content'][0]['value']
except KeyError:
description = entry['summary']
if entry.get('updated_parsed'):
last_update = get_dicttime(entry.updated_parsed)
else:
last_update = get_dicttime(datetime.datetime.now().timetuple())
if entry.get('published_parsed', False):
published = get_dicttime(entry.published_parsed)
else:
published = get_dicttime(datetime.datetime.now().timetuple())
storage.add_story({
'title': title,
'link': link,
'description': description,
'published': published,
'last_update': last_update,
'feed_id': feed['_id'],
'read': False})
def add(self, url):
url = url.strip()
feed = feedparser.parse(url)
# Bad feed
if not feed.feed.get('title', False):
return {'success': False, 'output':'Bad feed'}
title = feed.feed['title']
feed_id = backend.get_feed_by_title(title)
if not feed_id:
if feed.feed.get('updated_parsed'):
feed_update = get_dicttime(feed.feed.updated_parsed)
elif feed.get('updated_parsed'):
feed_update = get_dicttime(feed.updated_parsed)
elif feed.feed.get('published_parsed'):
feed_update = get_dicttime(feed.feed.published_parsed)
elif feed.get('published_parsed'):
feed_update = get_dicttime(feed.published_parsed)
else:
return {'success':False, 'output':'Parsing error'}
feed_id = backend.add_feed({'url':url, 'title': title, 'last_update': feed_update})
else:
return {'success':False, 'output':'Feed already exists'}
retriever = Retriever(feed)
retriever.start()
return {
'success': True,
'title': title,
'feed_id': feed_id,
'output': 'Feed added',
'counter': len(feed['entries'])}
def unread(self, story_id):
story = storage.get_story_by_id(story_id)
if not story['read']:
story['published'] = get_dicttime(story['published'])
story['last_update'] = get_dicttime(story['last_update'])
return {'success': False, 'output': 'Story already unreaded'}
story['read'] = False
storage.update_story(story['_id'], copy.copy(story))
story['published'] = get_dicttime(story['published'])
story['last_update'] = get_dicttime(story['last_update'])
return {'success': True, 'content': story}
def unread(self, story_id):
story = storage.get_story_by_id(story_id)
if not story['read']:
story['published'] = get_dicttime(story['published'])
story['last_update'] = get_dicttime(story['last_update'])
return {'success': False, 'output': 'Story already unreaded'}
story['read'] = False
storage.update_story(story['_id'], copy.copy(story))
story['published'] = get_dicttime(story['published'])
story['last_update'] = get_dicttime(story['last_update'])
return {'success': True, 'content': story}
def get(self, feed_id=False, feed_type="combined-feed", order_type="user", start=0, stop=50):
# Special feeds
if not feed_id:
if feed_type == "combined-feed":
if order_type == "user" or order_type not in ['unreaded', 'published']:
order_type = storage.get_feed_setting('combined-feed' , 'ordering')
if not order_type:
storage.set_feed_setting('combined-feed', 'ordering', 'unreaded')
order_type = 'unreaded'
else:
order_type = order_type['value']
elif feed_type == "stared-feed":
if order_type == "user" or order_type not in ['unreaded', 'published']:
order_type = storage.get_feed_setting('stared-feed', 'ordering')
if not order_type:
storage.set_feed_setting('stared-feed', 'ordering', 'unreaded')
order_type = 'unreaded'
else:
order_type = order_type['value']
else:
raise Exception('Unknown feed type (%s)' % feed_type)
# Normal feed
else:
if order_type == "user" or order_type not in ['unreaded', 'published']:
order_type = storage.get_feed_setting(feed_id, 'ordering')
if not order_type:
storage.set_feed_setting(feed_id, 'ordering', 'unreaded')
order_type = 'unreaded'
else:
order_type = order_type['value']
# Get stories
if not feed_id:
if feed_type == "combined-feed":
stories = storage.all_stories(order_type, start, stop)
elif feed_type == "stared-feed":
stories = storage.all_stared(order_type, start, stop)
else:
raise Exception('Unknown feed type (%s)' % feed_type)
else:
stories = storage.get_stories(feed_id, order_type, start, stop)
length = storage.get_feed_unread_count(feed_id)
count = storage.get_feed_count(feed_id)
res = []
for story in stories:
story['last_update'] = get_dicttime(story['last_update'])
story['published'] = get_dicttime(story['published'])
res.append(story)
return {'entries': res, 'ordering': order_type, 'detail': {'start': start, 'stop': stop, 'length': length, 'count':count}}
def run(self):
self.data = feedparser.parse(self.feed['url'])
local_update = get_datetime(self.feed['last_update'])
remote_update = False
if self.data.feed.get('updated_parsed'):
remote_update = get_datetime(self.data.feed.updated_parsed)
remote_update_raw = get_dicttime(self.data.feed.updated_parsed)
if self.data.get('updated_parsed'):
if remote_update:
if get_datetime(self.data.updated_parsed) > remote_update:
remote_update = get_datetime(self.data.updated_parsed)
remote_update_raw = get_dicttime(self.data.updated_parsed)
if self.data.feed.get('published_parsed'):
if remote_update:
if get_datetime(self.data.feed.published_parsed) > remote_update:
remote_update = get_datetime(self.data.feed.published_parsed)
remote_update_raw = get_dicttime(self.data.feed.published_parsed)
if self.data.get('published_parsed'):
if remote_update:
if get_datetime(self.data.published_parsed > remote_update):
remote_update = get_datetime(self.data.published_parsed)
remote_update_raw = get_dicttime(self.data.published_parsed)
if not remote_update:
remote_update = datetime.datetime.now()
remote_update_raw = get_dicttime(remote_update.timetuple())
if remote_update > local_update:
print(':: %s is outdated' % self.feed['title'].encode('utf-8'))
readed = []
for entry in storage.get_stories(self.feed['_id']):
if entry['read']:
readed.append(entry['title'])
storage.remove_story(entry['_id'])
retriever = Retriever(self.data)
retriever.start()
retriever.join()
for entry in readed:
if storage.get_story_by_title(entry):
entry = storage.get_story_by_title(entry)
entry['read'] = True
storage.update_story(entry['_id'], copy.copy(entry))
self.feed['last_update'] = remote_update_raw
storage.update_feed(self.feed_id, self.feed)
def add(self, url):
feed_guesser = self.get_feed(url)
if feed_guesser['success']:
feed, url = feed_guesser['output']
else:
return feed_guesser
# Bad feed
if feed.version == '' or not feed.feed.get('title'):
return {'success': False, 'output': 'Bad feed'}
title = feed.feed['title']
feed_id = storage.get_feed_by_title(title)
if not feed_id:
if feed.feed.get('updated_parsed'):
feed_update = get_dicttime(feed.feed.updated_parsed)
elif feed.get('updated_parsed'):
feed_update = get_dicttime(feed.updated_parsed)
elif feed.feed.get('published_parsed'):
feed_update = get_dicttime(feed.feed.published_parsed)
elif feed.get('published_parsed'):
feed_update = get_dicttime(feed.published_parsed)
else:
feed_update = get_dicttime(datetime.datetime.now().timetuple())
feed_id = storage.add_feed({'url': url,
'title': title,
'last_update': feed_update})
else:
return {'success': False, 'output': 'Feed already exists'}
retriever = Retriever(feed, do_retention=False)
retriever.start()
return {
'success': True,
'title': title,
'url': url,
'feed_id': feed_id,
'output': 'Feed added',
'counter': len(feed['entries'])}
def read(self, story_id):
"""
Return story content, set it at readed state and give
previous read state for counter
"""
story = storage.get_story_by_id(story_id)
if story['read']:
story['published'] = get_dicttime(story['published'])
story['last_update'] = get_dicttime(story['last_update'])
return {'success': False,
'output': 'Story already readed',
'content': story}
# Save read state before update it for javascript counter in UI
story['read'] = True
storage.update_story(story['_id'], copy.copy(story))
story['published'] = get_dicttime(story['published'])
story['last_update'] = get_dicttime(story['last_update'])
return {'success': True, 'content': story}
def run(self):
# This feed comes from database
feed = storage.get_feed_by_title(self.title)
for entry in self.data:
title = entry['title']
link = entry['link']
if storage.get_story_by_title(title):
storage.remove_story(storage.get_story_by_title(title)['_id'])
try:
description = entry['content'][0]['value']
except KeyError:
description = entry['summary']
if entry.get('updated_parsed'):
last_update = get_dicttime(entry.updated_parsed)
else:
last_update = get_dicttime(datetime.datetime.now().timetuple())
if entry.get('published_parsed', False):
published = get_dicttime(entry.published_parsed)
published_datetime = get_datetime(entry.published_parsed)
else:
published = get_dicttime(datetime.datetime.now().timetuple())
published_datetime = datetime.datetime.now()
if self.do_retention:
delta = datetime.datetime.now() - published_datetime
if delta.days > int(config.get('worker', 'retention')):
continue
storage.add_story({
'title': title,
'link': link,
'description': description,
'published': published,
'last_update': last_update,
'feed_id': feed['_id'],
'read': False})
def add(self, url):
url = url.strip()
feed = feedparser.parse(url)
# Bad feed
if not feed.feed.get('title', False):
return {'success': False, 'output': 'Bad feed'}
title = feed.feed['title']
feed_id = backend.get_feed_by_title(title)
if not feed_id:
if feed.feed.get('updated_parsed'):
feed_update = get_dicttime(feed.feed.updated_parsed)
elif feed.get('updated_parsed'):
feed_update = get_dicttime(feed.updated_parsed)
elif feed.feed.get('published_parsed'):
feed_update = get_dicttime(feed.feed.published_parsed)
elif feed.get('published_parsed'):
feed_update = get_dicttime(feed.published_parsed)
else:
return {'success': False, 'output': 'Parsing error'}
feed_id = backend.add_feed({
'url': url,
'title': title,
'last_update': feed_update
})
else:
return {'success': False, 'output': 'Feed already exists'}
retriever = Retriever(feed)
retriever.start()
return {
'success': True,
'title': title,
'feed_id': feed_id,
'output': 'Feed added',
'counter': len(feed['entries'])
}
def run(self):
self.data = feedparser.parse(self.feed['url'])
local_update = get_datetime(self.feed['last_update'])
if self.data.feed.get('updated_parsed'):
remote_update = get_datetime(self.data.feed.updated_parsed)
remote_update_raw = get_dicttime(self.data.feed.updated_parsed)
elif self.data.get('updated_parsed'):
remote_update = get_datetime(self.data.updated_parsed)
remote_update_raw = get_dicttime(self.data.updated_parsed)
elif self.data.feed.get('published_parsed'):
remote_update = get_datetime(self.data.feed.published_parsed)
remote_update_raw = get_dicttime(self.data.feed.published_parsed)
elif self.data.get('published_parsed'):
remote_update = get_datetime(self.data.published_parsed)
remote_update_raw = get_dicttime(self.data.published_parsed)
else:
return
if remote_update > local_update:
print(':: %s is outdated' % self.feed['title'])
readed = []
for entry in backend.get_stories(self.feed['_id']):
if entry['read']:
readed.append(entry['title'])
backend.remove_story(entry['_id'])
retriever = Retriever(self.data)
retriever.start()
retriever.join()
for entry in readed:
if backend.get_story_by_title(entry):
entry = backend.get_story_by_title(entry)
entry['read'] = True
backend.update_story(entry['_id'], copy.copy(entry))
self.feed['last_update'] = remote_update_raw
backend.update_feed(self.feed_id, self.feed)
def run(self):
self.data = feedparser.parse(self.feed['url'])
local_update = get_datetime(self.feed['last_update'])
if self.data.feed.get('updated_parsed'):
remote_update = get_datetime(self.data.feed.updated_parsed)
remote_update_raw = get_dicttime(self.data.feed.updated_parsed)
elif self.data.get('updated_parsed'):
remote_update = get_datetime(self.data.updated_parsed)
remote_update_raw = get_dicttime(self.data.updated_parsed)
elif self.data.feed.get('published_parsed'):
remote_update = get_datetime(self.data.feed.published_parsed)
remote_update_raw = get_dicttime(self.data.feed.published_parsed)
elif self.data.get('published_parsed'):
remote_update = get_datetime(self.data.published_parsed)
remote_update_raw = get_dicttime(self.data.published_parsed)
else:
return
if remote_update > local_update:
print(':: %s is outdated' % self.feed['title'])
readed = []
for entry in backend.get_stories(self.feed['_id']):
if entry['read']:
readed.append(entry['title'])
backend.remove_story(entry['_id'])
retriever = Retriever(self.data)
retriever.start()
retriever.join()
for entry in readed:
if backend.get_story_by_title(entry):
entry = backend.get_story_by_title(entry)
entry['read'] = True
backend.update_story(entry['_id'], copy.copy(entry))
self.feed['last_update'] = remote_update_raw
backend.update_feed(self.feed_id, self.feed)
def read(self, story_id):
"""
Return story content, set it at readed state and give
previous read state for counter
"""
story = storage.get_story_by_id(story_id)
if story['read']:
story['published'] = get_dicttime(story['published'])
story['last_update'] = get_dicttime(story['last_update'])
return {
'success': False,
'output': 'Story already readed',
'content': story
}
# Save read state before update it for javascript counter in UI
story['read'] = True
storage.update_story(story['_id'], copy.copy(story))
story['published'] = get_dicttime(story['published'])
story['last_update'] = get_dicttime(story['last_update'])
return {'success': True, 'content': story}
def get(self,
feed_id=False,
feed_type="combined-feed",
order_type="user",
start=0,
stop=50):
# Special feeds
if not feed_id:
if feed_type == "combined-feed":
if order_type == "user" or order_type not in [
'unreaded', 'published'
]:
order_type = storage.get_feed_setting(
'combined-feed', 'ordering')
if not order_type:
storage.set_feed_setting('combined-feed', 'ordering',
'unreaded')
order_type = 'unreaded'
else:
order_type = order_type['value']
elif feed_type == "stared-feed":
if order_type == "user" or order_type not in [
'unreaded', 'published'
]:
order_type = storage.get_feed_setting(
'stared-feed', 'ordering')
if not order_type:
storage.set_feed_setting('stared-feed', 'ordering',
'unreaded')
order_type = 'unreaded'
else:
order_type = order_type['value']
else:
raise Exception('Unknown feed type (%s)' % feed_type)
# Normal feed
else:
if order_type == "user" or order_type not in [
'unreaded', 'published'
]:
order_type = storage.get_feed_setting(feed_id, 'ordering')
if not order_type:
storage.set_feed_setting(feed_id, 'ordering', 'unreaded')
order_type = 'unreaded'
else:
order_type = order_type['value']
# Get stories
if not feed_id:
if feed_type == "combined-feed":
stories = storage.all_stories(order_type, start, stop)
elif feed_type == "stared-feed":
stories = storage.all_stared(order_type, start, stop)
else:
raise Exception('Unknown feed type (%s)' % feed_type)
else:
stories = storage.get_stories(feed_id, order_type, start, stop)
length = storage.get_feed_unread_count(feed_id)
res = []
for story in stories:
story['last_update'] = get_dicttime(story['last_update'])
story['published'] = get_dicttime(story['published'])
res.append(story)
return {
'entries': res,
'ordering': order_type,
'detail': {
'start': start,
'stop': stop,
'length': length
}
}
