def parse(self):
"""
Parses raw data into Entries
"""
if self.data is None and self.raw_data:
try:
self.data = self.parse_raw_data(self.raw_data)
except ParserException as e:
app.logger.exception(
u'Failed to parse notification for Feed {0}, Message: {1}, Raw_data: {2}'
.format(self.feed, str(e)), self.raw_data)
return
elif self.data is None and self.raw_data is None:
raise ParserException('No data passed into parser')
# Wordpress hubs don't send correct feed details
if not self.is_wordpress_hub(self.feed.hub):
self.feed = self.update_feed_details(self.data, self.feed)
self.feed = self.update_feed_version(self.data, self.feed)
db.session.add(self.feed)
items = self.get_items(self.data)
if not items:
app.logger.info(u'No items to parse for Feed {0}'
.format(self.feed))
db.session.commit()
return
self.create_item_parsers(items)
self.add_entries_to_item_parsers()
app.logger.info(u'Parsing {0} items'.format(len(self.item_parsers)))
self.parse_items(self.item_parsers)
# Removed for performance reasons. Author RSS is updated on request.
# if self.authors:
# self.update_authors(self.authors)
# Send entries to SubscriptionHandler
if self.entries:
new_entries = self.new_entries(self.entries)
app.logger.info(
u'Sending entries_added signal for {0} new entries: {1}'
.format(len(new_entries), new_entries))
entries_added.send(self, entries=new_entries)
db.session.commit()
app.logger.info(
u'Parsed {0} items, added/updated {1} entries and {2} authors'.format(
len(items), len(self.entries), len(self.authors)))
return
def test_listen_for_entries_added(subhandler, session):
user = UserFactory(active=True)
author = AuthorFactory()
entry = EntryFactory()
entry.authors.append(author)
sub = SubscriptionFactory(user=user, author=author, active=True)
sub.add_period(PERIOD.IMMEDIATE)
db.session.add(sub)
db.session.commit()
class TestSender:
def __init__(self, entries):
self.entries = entries
with mail.record_messages() as outbox:
entries_added.connect(when_entries_added)
sender = TestSender([entry])
assert len(sender.entries) == 1
entries_added.send(sender, entries=[entry])
assert len(outbox) == 1
def parse(self):
"""
Parses raw data into Entries
"""
if not self.data:
raise ParserException("No data passed into parser")
if not self.parsed:
try:
self.data = self.parse_raw_data(self.data, self.encoding, self.headers)
except ParserException as e:
app.logger.exception(
"Failed to parse notification for %s, " "Message: %s, Raw_data: %s",
self.feed,
e,
self.data,
)
return
# Wordpress hubs don't send correct feed details
if not self.is_wordpress_hub(self.feed.hub):
self.update_feed_details(self.data, self.feed)
self.update_feed_version(self.data, self.feed)
# db.session.add(self.feed)
items = self.get_items(self.data)
if not items:
app.logger.info("No items to parse for %s", self.feed)
db.session.commit()
return
# Create an item parser for each item
self.create_item_parsers(items)
# Search for existing entries for each item guid, and add them
# to the corresponding item parser.
self.add_entries_to_item_parsers()
# Parse items
self.parse_items(self.item_parsers)
db.session.commit()
# Update author RSS feeds
if self.authors:
app.logger.info(
"Sending authors-entries-added signal for %d authors", len(self.authors)
)
authors_entries_added.send(self, authors=self.authors)
# Send entries to SubscriptionHandler
new_entries = self.new_entries(self.entries)
# Update FeedStats
self.update_stats(new_entries, len(items))
if new_entries:
app.logger.info(
"Sending entries_added signal for %d new entries", len(new_entries)
)
entries_added.send(self, entries=new_entries)
db.session.commit()
app.logger.info(
"PARSED: items=%d new=%d updated=%d authors=%d",
len(items),
self.len_new_entries,
self.len_updated_entries,
len(self.authors),
)
return
