def parse_buzzs(atom, googleprofile, logger):
messenger = StompMessenger()
siteuser = googleprofile.siteuser
if siteuser.usersettings.usetag:
tagre = re.compile(siteuser.usersettings.mark)
feedparse_result = feedparser.parse(atom)
newest_date = None
newentries = []
logger.info("XXX 1")
for entry in feedparse_result.entries:
logger.info("XXX 2")
try:
d = entry.published_parsed
except AttributeError:
# Sometimes posts doesnt have published field and are not published on Buzz :-?
logger.info("XXX 3")
continue
logger.info("XXX 4")
#pprint(entry)
date_buzz = datetime(d[0], d[1], d[2], d[3], d[4], d[5])
if ( googleprofile.firsttime and date_buzz > googleprofile.created_at) or \
((not googleprofile.firsttime) and date_buzz > googleprofile.lastbuffread_at): \
# Get links/photos (if there is any)
linkscontent = []
logger.info('XXX 5')
linkslist = []
buzzlink = comments = links = linkslist_str = ''
logger.info("XXX 6")
for link in entry.links:
if link.rel == u'enclosure':
if link.href not in linkscontent: # avoid duplicates
linkscontent.append(link.href)
linkslist.append({'href': link.href, 'title': link.title, 'type': link.type})
linkslist_str += link.href
elif link.rel == u'alternate':
buzzlink = link.href
elif link.rel == u'replies':
comments = link.href
links = '|'.join(linkslist_str)
buzzcontent = entry.content[0].value
logger.info('XXX 7')
if settings.CMDRE.search(buzzcontent):
# Is a command, execute it but don't save the buzz
logger.info(u'parsing command for: %s' % buzzcontent)
execute_command(buzzcontent, siteuser, messenger)
logger.info('XXX 8')
continue
# User want to publish only buzzs with tag? is there a tag?
if siteuser.usersettings.paused or (siteuser.usersettings.usetag and not tagre.search(buzzcontent)):
logger.info('XXX Ignoring, paused or with usetag without tag')
continue
newbuzz = Buzz(buzzid = entry.id,
content = buzzcontent,
links = links,
user = siteuser,
buzzlink = buzzlink)
if comments != '':
newbuzz.commentslink = comments
msg = 'Saving new buzz: ' + unicode(buzzcontent).encode('utf-8')
logger.info(msg)
newbuzz.save()
# Send a message for the listener.post_buzz2twitter to post the status
logger.info('sending message to queue publish__%d' % newbuzz.id)
messenger.sendmessage('/queue/listener', 'publish__%d' % newbuzz.id)
if newest_date == None or date_buzz > newest_date:
newest_date = date_buzz
if len(feedparse_result.entries) == 0:
logger.warning('There are 0 entries in the feed, could be a parse error')
if feedparse_result.has_key('bozo_exception'):
logger.warning('bozo_exception is: ' + str(feedparse_result.bozo_exception))
else:
# Update the date of the last buzz read and reset the retries
logger.debug('new lastbuzzread_at is: ' + str(newest_date))
if newest_date != None:
googleprofile.firsttime = False
googleprofile.lastbuffread_at = newest_date
googleprofile.retries = 0
googleprofile.save()
def processnotification(self, notif_id):
"""
Parse an Atom notificaton received from the pubsubhubbub callback
"""
msg = 'listener.processnotification: msg received: ' + str(notif_id)
self.logger.debug(msg)
notif = Notification.objects.get(id=int(notif_id))
googleprofile = notif.googleprofile
siteuser = googleprofile.siteuser
#self.logger.debug('XXX 1')
if siteuser.usersettings.usetag:
tagre = re.compile(siteuser.usersettings.mark)
feedparse_result = feedparser.parse(notif.text.encode('utf-8'))
newest_date = None
newentries = []
#self.logger.debug('XXX 2')
ld = self.logger.debug # XXX
for entry in feedparse_result.entries:
#self.logger.debug('XXX 3')
try:
d = entry.published_parsed
except AttributeError:
# Sometimes posts doesnt have published field and are not published on Buzz :-?
#ld('XXX 4')
continue
#pprint(entry)
date_buzz = datetime(d[0], d[1], d[2], d[3], d[4], d[5])
# Note: if the user is new lastbuffread_at == created_at
ld('XXX date_buzz: %s lastbuffread_at: %s' %
(str(date_buzz), str(googleprofile.lastbuffread_at)))
if date_buzz > googleprofile.lastbuffread_at:
#ld('XXX 5')
# Get links/photos (if there is any)
linkscontent = [] # Only the URL
linkslist = [] # Dict with all the data
buzzlink = comments = links = ''
for link in entry.links:
if link.rel == u'enclosure':
if link.href not in linkscontent: # avoid duplicates
linkscontent.append(link.href)
linkslist.append({
'href': link.href,
'title': link.get('title', ''),
'type': link.type
})
elif link.rel == u'alternate':
buzzlink = link.href
elif link.rel == u'replies':
comments = link.href
links = '|'.join(linkscontent)
buzzcontent = shortenlinks_removetags(entry.content[0].value,
self.recompiled,
self.bitlyapi)
#ld('XXX 6')
if settings.CMDRE.search(buzzcontent):
#ld('XXX 7')
# Is a command, execute it but don't save the buzz
self.logger.info(u'parsing command for: %s' % buzzcontent)
execute_command(buzzcontent, siteuser, self.messenger)
continue
# User want to publish only buzzs with tag? is there a tag?
if siteuser.usersettings.paused or (
siteuser.usersettings.usetag
and not tagre.search(buzzcontent)):
#ld('XXX 8')
continue
newbuzz = Buzz(buzzid=entry.id,
content=buzzcontent,
links=links,
user=siteuser,
buzzlink=buzzlink)
if comments != '':
newbuzz.commentslink = comments
msg = u'Saving new buzz: ' + buzzcontent
self.logger.info(msg)
try:
#ld('XXX 9')
newbuzz.save()
except IntegrityError:
# User probably issued a 'cancel' command just before this message
self.logger.warning(
'Received IntegrityError for user while saving Buzz, probably already deleted'
)
continue
#ld('XXX 10')
# Send a message for the listener.post_buzz2twitter to post the status
self.logger.info('sending message to queue publish__%d' %
newbuzz.id)
self.messenger.sendmessage('/queue/listener',
'publish__%d' % newbuzz.id)
#ld('XXX 11')
if newest_date == None or date_buzz > newest_date:
newest_date = date_buzz
if len(feedparse_result.entries) == 0:
self.logger.warning(
'There are 0 entries in the feed, could be a parse error')
if feedparse_result.has_key('bozo_exception'):
self.logger.warning('bozo_exception is: ' +
str(feedparse_result.bozo_exception))
else:
# Update the date of the last buzz read and reset the retries
self.logger.debug('new lastbuzzread_at is: ' + str(newest_date))
if newest_date != None:
googleprofile.firsttime = False
googleprofile.lastbuffread_at = newest_date
googleprofile.retries = 0
googleprofile.save()
def processnotification(self, notif_id):
"""
Parse an Atom notificaton received from the pubsubhubbub callback
"""
msg = 'listener.processnotification: msg received: ' + str(notif_id)
self.logger.debug(msg)
notif = Notification.objects.get(id = int(notif_id))
googleprofile = notif.googleprofile
siteuser = googleprofile.siteuser
#self.logger.debug('XXX 1')
if siteuser.usersettings.usetag:
tagre = re.compile(siteuser.usersettings.mark)
feedparse_result = feedparser.parse(notif.text.encode('utf-8'))
newest_date = None
newentries = []
#self.logger.debug('XXX 2')
ld = self.logger.debug # XXX
for entry in feedparse_result.entries:
#self.logger.debug('XXX 3')
try:
d = entry.published_parsed
except AttributeError:
# Sometimes posts doesnt have published field and are not published on Buzz :-?
#ld('XXX 4')
continue
#pprint(entry)
date_buzz = datetime(d[0], d[1], d[2], d[3], d[4], d[5])
# Note: if the user is new lastbuffread_at == created_at
ld('XXX date_buzz: %s lastbuffread_at: %s' % (str(date_buzz), str(googleprofile.lastbuffread_at)))
if date_buzz > googleprofile.lastbuffread_at:
#ld('XXX 5')
# Get links/photos (if there is any)
linkscontent = [] # Only the URL
linkslist = [] # Dict with all the data
buzzlink = comments = links = ''
for link in entry.links:
if link.rel == u'enclosure':
if link.href not in linkscontent: # avoid duplicates
linkscontent.append(link.href)
linkslist.append({'href': link.href, 'title': link.get('title', ''), 'type': link.type})
elif link.rel == u'alternate':
buzzlink = link.href
elif link.rel == u'replies':
comments = link.href
links = '|'.join(linkscontent)
buzzcontent = shortenlinks_removetags(entry.content[0].value, self.recompiled, self.bitlyapi)
#ld('XXX 6')
if settings.CMDRE.search(buzzcontent):
#ld('XXX 7')
# Is a command, execute it but don't save the buzz
self.logger.info(u'parsing command for: %s' % buzzcontent)
execute_command(buzzcontent, siteuser, self.messenger)
continue
# User want to publish only buzzs with tag? is there a tag?
if siteuser.usersettings.paused or (siteuser.usersettings.usetag and not tagre.search(buzzcontent)):
#ld('XXX 8')
continue
newbuzz = Buzz(buzzid = entry.id,
content = buzzcontent,
links = links,
user = siteuser,
buzzlink = buzzlink)
if comments != '':
newbuzz.commentslink = comments
msg = u'Saving new buzz: ' + buzzcontent
self.logger.info(msg)
try:
#ld('XXX 9')
newbuzz.save()
except IntegrityError:
# User probably issued a 'cancel' command just before this message
self.logger.warning('Received IntegrityError for user while saving Buzz, probably already deleted')
continue
#ld('XXX 10')
# Send a message for the listener.post_buzz2twitter to post the status
self.logger.info('sending message to queue publish__%d' % newbuzz.id)
self.messenger.sendmessage('/queue/listener', 'publish__%d' % newbuzz.id)
#ld('XXX 11')
if newest_date == None or date_buzz > newest_date:
newest_date = date_buzz
if len(feedparse_result.entries) == 0:
self.logger.warning('There are 0 entries in the feed, could be a parse error')
if feedparse_result.has_key('bozo_exception'):
self.logger.warning('bozo_exception is: ' + str(feedparse_result.bozo_exception))
else:
# Update the date of the last buzz read and reset the retries
self.logger.debug('new lastbuzzread_at is: ' + str(newest_date))
if newest_date != None:
googleprofile.firsttime = False
googleprofile.lastbuffread_at = newest_date
googleprofile.retries = 0
googleprofile.save()
def fetchuserfeed(self, message):
"""
Parse a user Buzz Atom feed
"""
profile = None
msg = 'listener.fetchuserfeed: msg received: ' + message
self.logger.debug(msg)
try:
siteuserid = int(message.strip())
siteuser = SiteUser.objects.get(id=siteuserid)
if siteuser.usersettings.usetag:
tagre = re.compile(siteuser.usersettings.mark)
profile = siteuser.googleprofile
try:
(newentries, newest_date) = profile.get_new_entries()
except BuzzFeedParsingException, e:
msg = 'Exception parsing feed for user %d: %s' % (siteuser.id,
str(e))
self.logger.debug(msg)
return
self.logger.debug('There are %d new entries for this user' %
len(newentries))
if len(newentries) > 0:
for entry in newentries:
buzzcontent = shortenlinks_removetags(
entry['content'].value, self.recompiled, self.bitlyapi)
print 'XXXXXXXXXXXXXX buzzxcontent: ' + str(buzzcontent)
if self.cmdre.search(buzzcontent):
# Is a command, execute it but don't save the buzz
self.logger.info(u'parsing command for: %s' %
buzzcontent)
execute_command(buzzcontent, siteuser, self.messenger)
continue
# User want to publish only buzzs with tag? is there a tag?
if siteuser.usersettings.paused or (
siteuser.usersettings.usetag
and not tagre.search(buzzcontent)):
continue
newbuzz = Buzz(buzzid=entry['id'],
content=buzzcontent,
links=entry['links'],
user=siteuser,
buzzlink=entry['buzzlink'])
if entry.has_key('comments'):
newbuzz.commentslink = entry['comments']
msg = u'Saving new buzz: ' + buzzcontent
self.logger.info(msg)
newbuzz.save()
# Send a message for the listener.post_buzz2twitter to post the status
self.messenger.sendmessage('/queue/listener',
'publish__%d' % newbuzz.id)
profile.firsttime = False
self.logger.info('Asignando a lastbuffread_at la fecha: ' +
str(newest_date))
profile.lastbuffread_at = newest_date
profile.retries = 0
profile.save()
def fetchuserfeed(self, message):
"""
Parse a user Buzz Atom feed
"""
profile = None
msg = 'listener.fetchuserfeed: msg received: ' + message
self.logger.debug(msg)
try:
siteuserid = int(message.strip())
siteuser = SiteUser.objects.get(id=siteuserid)
if siteuser.usersettings.usetag:
tagre = re.compile(siteuser.usersettings.mark)
profile = siteuser.googleprofile
try:
(newentries, newest_date) = profile.get_new_entries()
except BuzzFeedParsingException, e:
msg = 'Exception parsing feed for user %d: %s' % (siteuser.id, str(e))
self.logger.debug(msg)
return
self.logger.debug('There are %d new entries for this user' % len(newentries))
if len(newentries) > 0:
for entry in newentries:
buzzcontent = shortenlinks_removetags(entry['content'].value, self.recompiled, self.bitlyapi)
print 'XXXXXXXXXXXXXX buzzxcontent: ' + str(buzzcontent)
if self.cmdre.search(buzzcontent):
# Is a command, execute it but don't save the buzz
self.logger.info(u'parsing command for: %s' % buzzcontent)
execute_command(buzzcontent, siteuser, self.messenger)
continue
# User want to publish only buzzs with tag? is there a tag?
if siteuser.usersettings.paused or (siteuser.usersettings.usetag and not tagre.search(buzzcontent)):
continue
newbuzz = Buzz(buzzid = entry['id'],
content = buzzcontent,
links = entry['links'],
user = siteuser,
buzzlink = entry['buzzlink'])
if entry.has_key('comments'):
newbuzz.commentslink = entry['comments']
msg = u'Saving new buzz: ' + buzzcontent
self.logger.info(msg)
newbuzz.save()
# Send a message for the listener.post_buzz2twitter to post the status
self.messenger.sendmessage('/queue/listener', 'publish__%d' % newbuzz.id)
profile.firsttime = False
self.logger.info('Asignando a lastbuffread_at la fecha: ' + str(newest_date))
profile.lastbuffread_at = newest_date
profile.retries = 0
profile.save()
