def send_confirmations (self): # get subscriptions in need of confirmation subs = self.db.getEmailSubscriptionsForConfirmation () self.log('Sending confirmations to %s subscriptions ' % (len(subs)), log.INFO) for sub in subs: msg = MIMEMultipart('alternative') msg['Subject'] = '[SkyTruth Alerts] Subscription Confirmation' msg['From'] = '*****@*****.**' msg['To'] = sub['email'] map_size = 256 params = {} params ['confirm_url'] = "http://alerts.skytruth.org/subscribe?sid=%s" % sub['id'] params ['static_map_url'] = self.get_static_map_url (sub, map_size) params ['static_map_width'] = map_size params ['static_map_height'] = map_size msg_templates = self.get_message_templates () msg.attach(MIMEText(msg_templates['text']['confirm'].substitute (params), 'plain')) msg.attach(MIMEText(msg_templates['html']['confirm'].substitute (params), 'html')) self.log('sending confirmation email to %s for subscription %s' % (msg['To'], sub['id']), log.INFO) self.send_email (msg['From'], msg['To'], msg) # update subscription status self.db.updateEmailSubscription (sub['id'], {'last_email_sent':format_datetime(datetime.now())})
def send_updates(self): # get subscriptions in need of update subs = self.db.getEmailSubscriptionsForUpdate() self.log('Sending updates to %s subscriptions ' % (len(subs)), log.INFO) for sub in subs: # get RSS feed self.log('reading rss feed: %s' % (sub['rss_url']), log.INFO) rss_complete_url = self.complete_url(sub) feed_data = feedparser.parse(rss_complete_url) # construct message msg_parts = self.compose_message(sub, feed_data) # send email if msg_parts: # used the last example here: #http://docs.python.org/library/email-examples.html#email-examples msg = MIMEMultipart('alternative') msg['Subject'] = msg_parts['subject'] msg['From'] = '*****@*****.**' msg['To'] = sub['email'] # The email lib has trouble with unocode, so we encode as ascii msg.attach( MIMEText(msg_parts['text'].encode('ascii', 'replace'), 'plain')) msg.attach( MIMEText( msg_parts['html'].encode('ascii', 'xmlcharrefreplace'), 'html')) self.log( 'sending email to %s with %s new items' % (msg['To'], msg_parts['item_count']), log.INFO) self.send_email(msg['From'], msg['To'], msg) # update subscription status self.db.updateEmailSubscription( sub['id'], { 'last_email_sent': format_datetime(datetime.now()), 'last_item_updated': msg_parts['last_item_updated'] })
def send_confirmations(self): # get subscriptions in need of confirmation subs = self.db.getEmailSubscriptionsForConfirmation() self.log('Sending confirmations to %s subscriptions ' % (len(subs)), log.INFO) for sub in subs: msg = MIMEMultipart('alternative') msg['Subject'] = '[SkyTruth Alerts] Subscription Confirmation' msg['From'] = '*****@*****.**' msg['To'] = sub['email'] map_size = 256 params = {} params[ 'confirm_url'] = "http://alerts.skytruth.org/subscribe?sid=%s" % sub[ 'id'] params['static_map_url'] = self.get_static_map_url(sub, map_size) params['static_map_width'] = map_size params['static_map_height'] = map_size msg_templates = self.get_message_templates() msg.attach( MIMEText(msg_templates['text']['confirm'].substitute(params), 'plain')) msg.attach( MIMEText(msg_templates['html']['confirm'].substitute(params), 'html')) self.log( 'sending confirmation email to %s for subscription %s' % (msg['To'], sub['id']), log.INFO) self.send_email(msg['From'], msg['To'], msg) # update subscription status self.db.updateEmailSubscription( sub['id'], {'last_email_sent': format_datetime(datetime.now())})
def send_updates (self): # get subscriptions in need of update subs = self.db.getEmailSubscriptionsForUpdate () self.log('Sending updates to %s subscriptions ' % (len(subs)), log.INFO) for sub in subs: # get RSS feed self.log('reading rss feed: %s' % (sub['rss_url']), log.INFO) rss_complete_url = self.complete_url(sub) feed_data = feedparser.parse(rss_complete_url) # construct message msg_parts = self.compose_message (sub, feed_data) # send email if msg_parts: # used the last example here: #http://docs.python.org/library/email-examples.html#email-examples msg = MIMEMultipart('alternative') msg['Subject'] = msg_parts['subject'] msg['From'] = '*****@*****.**' msg['To'] = sub['email'] # The email lib has trouble with unocode, so we encode as ascii msg.attach(MIMEText(msg_parts['text'].encode('ascii', 'replace'), 'plain')) msg.attach(MIMEText(msg_parts['html'].encode('ascii', 'xmlcharrefreplace'), 'html')) self.log('sending email to %s with %s new items' % (msg['To'], msg_parts['item_count']), log.INFO) self.send_email (msg['From'], msg['To'], msg) # update subscription status self.db.updateEmailSubscription (sub['id'], {'last_email_sent': format_datetime(datetime.now()), 'last_item_updated': msg_parts['last_item_updated']})
def process_item(self, task_id): feed = self.db.getRssFeeds(task_id) self.log('processsing rss feed %s (%s)' % (feed['id'], feed['url']), log.INFO) # parse feed feed_data = feedparser.parse( feed['url'] ) self.log('reading %s feed items' % (len(feed_data['items'])), log.INFO) # update last read time self.db.updateRssFeedLastRead (task_id) # For each item in feed for item in feed_data['items']: # if this item has not already been processed if self.db.rssFeedItemExists(item['id']): self.log('%s - feed item already exists - skipping' % (item['id']), log.INFO) continue # store the full item l=ItemLoader (RssFeedItem()) l.add_value ('item_id', item['id']) l.add_value ('content', psycopg2.Binary(pickle.dumps(item))) #l.add_value ('content', pickle.dumps(item)) l.add_value ('feed_id', task_id) yield l.load_item() #feed_entry_id = uuid.uuid5(uuid.NAMESPACE_URL, str(item ['id'])) feed_entry_id = self.db.uuid5_str(name=str(item ['id'])) l=ItemLoader (FeedEntry()) l.add_value ('id', feed_entry_id) l.add_value ('title', item['title']) # l.add_value ('updated', format_datetime(item['updated_parsed'])) l.add_value ('incident_datetime', format_datetime(item['updated_parsed'])) if 'content' in item: for c in item['content']: l.add_value ('content', c['value']) elif 'summary' in item: l.add_value ('content', item['summary']) embedded_fields = self.extractContentFields (l.get_output_value('content')) pt = embedded_fields.get('location') or item.get('georss_point') # print "em: '%s' geo: '%s'" % (embedded_fields.get('location'), item.get('georss_point')) if not pt: self.log('%s - No georeference found' % (item['id']), log.WARNING) continue pt = re.split ("[, ]+", pt) l.add_value ('lat', pt[0]) l.add_value ('lng', pt[1]) l.add_value ('kml_url', embedded_fields.get('kml') or '') for link in item['links']: if link['rel'] == 'alternate': l.add_value ('link', link['href']) l.add_value ('source_id', feed['source_id']) yield l.load_item() if 'tags' in item: for t in item['tags']: l=ItemLoader (FeedEntryTag()) l.add_value ('feed_entry_id', feed_entry_id) l.add_value ('tag', t['term']) l.add_value ('comment', t['label']) yield l.load_item() l=ItemLoader (FeedEntryTag()) l.add_value ('feed_entry_id', feed_entry_id) l.add_value ('tag', feed['tag']) yield l.load_item() # update task status self.item_completed (task_id)
def process_item(self, task_id): feed = self.db.getRssFeeds(task_id) self.log('processsing rss feed %s (%s)' % (feed['id'], feed['url']), log.INFO) # parse feed feed_data = feedparser.parse(feed['url']) self.log('reading %s feed items' % (len(feed_data['items'])), log.INFO) # update last read time self.db.updateRssFeedLastRead(task_id) # For each item in feed for item in feed_data['items']: # if this item has not already been processed if self.db.rssFeedItemExists(item['id']): self.log( '%s - feed item already exists - skipping' % (item['id']), log.INFO) continue # store the full item l = ItemLoader(RssFeedItem()) l.add_value('item_id', item['id']) l.add_value('content', psycopg2.Binary(pickle.dumps(item))) #l.add_value ('content', pickle.dumps(item)) l.add_value('feed_id', task_id) yield l.load_item() #feed_entry_id = uuid.uuid5(uuid.NAMESPACE_URL, str(item ['id'])) feed_entry_id = self.db.uuid5_str(name=str(item['id'])) l = ItemLoader(FeedEntry()) l.add_value('id', feed_entry_id) l.add_value('title', item['title']) # l.add_value ('updated', format_datetime(item['updated_parsed'])) l.add_value('incident_datetime', format_datetime(item['updated_parsed'])) if 'content' in item: for c in item['content']: l.add_value('content', c['value']) elif 'summary' in item: l.add_value('content', item['summary']) embedded_fields = self.extractContentFields( l.get_output_value('content')) pt = embedded_fields.get('location') or item.get('georss_point') # print "em: '%s' geo: '%s'" % (embedded_fields.get('location'), item.get('georss_point')) if not pt: self.log('%s - No georeference found' % (item['id']), log.WARNING) continue pt = re.split("[, ]+", pt) l.add_value('lat', pt[0]) l.add_value('lng', pt[1]) l.add_value('kml_url', embedded_fields.get('kml') or '') for link in item['links']: if link['rel'] == 'alternate': l.add_value('link', link['href']) l.add_value('source_id', feed['source_id']) yield l.load_item() if 'tags' in item: for t in item['tags']: l = ItemLoader(FeedEntryTag()) l.add_value('feed_entry_id', feed_entry_id) l.add_value('tag', t['term']) l.add_value('comment', t['label']) yield l.load_item() l = ItemLoader(FeedEntryTag()) l.add_value('feed_entry_id', feed_entry_id) l.add_value('tag', feed['tag']) yield l.load_item() # update task status self.item_completed(task_id)