def feeds():
"Update feed entries"
#TODO: check update interval &
feeds = Feed.query.filter_by(update_lock=False).all()
for feed in feeds:
logger.debug("Processing feed: %s <%s>", feed.id, feed.feed_url)
if feed.update_interval is None:
update_interval = reader.settings.FEED_UPDATE_TTL
else:
update_interval = feed.update_interval
if feed.last_update_started is not None:
if feed.last_update_started + timedelta(minutes=update_interval) > datetime.now():
logger.info("This feed is recently fetched")
continue
logger.debug("Setting update lock")
feed.update_lock = True
feed.last_update_started = datetime.now()
FeedRepository.save(feed)
try:
logger.debug("Fetching XML data from: %s", feed.feed_url)
d = feedparser.parse(feed.feed_url, etag=feed.last_etag, modified=feed.last_modified, agent="Breakfast https://github.com/VN-Nerds/breakfast")
if not d.feed:
logger.debug("This feed has not been modified since last fetch")
else:
try:
last_published = d.feed.published_parsed if 'published_parsed' in d.feed else d.feed.updated_parsed if 'updated_parsed' in d.feed else None
last_published = datetime.fromtimestamp(mktime(last_published))
except Exception:
last_published = None
entry_ids = []
if last_published and feed.last_published and feed.last_published >= last_published:
logger.debug("This feed has not been updated since last fetch")
else:
feed.last_published = last_published
feed.last_etag = d.etag if 'etag' in d else ""
feed.last_modified = datetime.fromtimestamp(mktime(d.modified_parsed)) if 'modified_parsed' in d else None
logger.debug("Found %s entries", len(d))
if d.entries:
for entry in d.entries:
unique_id = uuid.uuid5(uuid.NAMESPACE_URL, str(entry.link))
e = Entry.query.filter_by(uuid=unique_id).first()
status = "exists"
if not e:
status = "done"
title = entry.title
link = entry.link
content = entry.content[0].value if 'content' in entry else entry.summary
try:
published = entry.published_parsed if 'published_parsed' in entry else entry.updated_parsed if 'updated_parsed' in entry else datetime.now()
published = datetime.fromtimestamp(mktime(published))
except:
published = None
author = entry.author if 'author' in entry else None
comments = entry.comments if 'comments' in entry else None
try:
e = EntryRepository.create(feed.id, title, link, unique_id, content, published, author, comments)
entry_ids.append(e.id)
except Exception as ex:
status = "error"
logger.error("Error: %s", ex)
traceback.print_exc()
else:
entry_ids.append(e.id)
logger.debug("Processing entry [%s] with uuid [%s].... %s", entry.title, unique_id, status)
# list of users that have current feed
if entry_ids:
user_feeds = UserFeed.query.filter_by(feed_id=feed.id).all()
logger.debug("Found %s subscribers for this feed", len(user_feeds))
for user_feed in user_feeds:
for entry_id in entry_ids:
status = "exists"
ue = UserEntry.query.filter_by(entry_id=entry_id, user_feed_id=user_feed.id, user_id=user_feed.user_id).first()
if not ue:
status = "done"
UserEntryRepository.create(user_feed.user_id, entry_id, user_feed.id)
logger.debug("Adding entry [%s] for user [%s]... %s", entry_id, user_feed.user_id, status)
feed.last_error = "" # clear error message
except Exception as ex:
logger.error("Error: %s", ex)
traceback.print_exc()
feed.last_error = str(ex)
logger.debug("Release update lock for: %s", feed.feed_url)
feed.update_lock = False
db.session.add(feed)
db.session.commit()