def send_confirmations (self):
# get subscriptions in need of confirmation
subs = self.db.getEmailSubscriptionsForConfirmation ()
self.log('Sending confirmations to %s subscriptions ' % (len(subs)), log.INFO)
for sub in subs:
msg = MIMEMultipart('alternative')
msg['Subject'] = '[SkyTruth Alerts] Subscription Confirmation'
msg['From'] = '*****@*****.**'
msg['To'] = sub['email']
map_size = 256
params = {}
params ['confirm_url'] = "http://alerts.skytruth.org/subscribe?sid=%s" % sub['id']
params ['static_map_url'] = self.get_static_map_url (sub, map_size)
params ['static_map_width'] = map_size
params ['static_map_height'] = map_size
msg_templates = self.get_message_templates ()
msg.attach(MIMEText(msg_templates['text']['confirm'].substitute (params), 'plain'))
msg.attach(MIMEText(msg_templates['html']['confirm'].substitute (params), 'html'))
self.log('sending confirmation email to %s for subscription %s' % (msg['To'], sub['id']), log.INFO)
self.send_email (msg['From'], msg['To'], msg)
# update subscription status
self.db.updateEmailSubscription (sub['id'], {'last_email_sent':format_datetime(datetime.now())})
def send_updates(self):
# get subscriptions in need of update
subs = self.db.getEmailSubscriptionsForUpdate()
self.log('Sending updates to %s subscriptions ' % (len(subs)),
log.INFO)
for sub in subs:
# get RSS feed
self.log('reading rss feed: %s' % (sub['rss_url']), log.INFO)
rss_complete_url = self.complete_url(sub)
feed_data = feedparser.parse(rss_complete_url)
# construct message
msg_parts = self.compose_message(sub, feed_data)
# send email
if msg_parts:
# used the last example here:
#http://docs.python.org/library/email-examples.html#email-examples
msg = MIMEMultipart('alternative')
msg['Subject'] = msg_parts['subject']
msg['From'] = '*****@*****.**'
msg['To'] = sub['email']
# The email lib has trouble with unocode, so we encode as ascii
msg.attach(
MIMEText(msg_parts['text'].encode('ascii', 'replace'),
'plain'))
msg.attach(
MIMEText(
msg_parts['html'].encode('ascii', 'xmlcharrefreplace'),
'html'))
self.log(
'sending email to %s with %s new items' %
(msg['To'], msg_parts['item_count']), log.INFO)
self.send_email(msg['From'], msg['To'], msg)
# update subscription status
self.db.updateEmailSubscription(
sub['id'], {
'last_email_sent': format_datetime(datetime.now()),
'last_item_updated': msg_parts['last_item_updated']
})
def send_confirmations(self):
# get subscriptions in need of confirmation
subs = self.db.getEmailSubscriptionsForConfirmation()
self.log('Sending confirmations to %s subscriptions ' % (len(subs)),
log.INFO)
for sub in subs:
msg = MIMEMultipart('alternative')
msg['Subject'] = '[SkyTruth Alerts] Subscription Confirmation'
msg['From'] = '*****@*****.**'
msg['To'] = sub['email']
map_size = 256
params = {}
params[
'confirm_url'] = "http://alerts.skytruth.org/subscribe?sid=%s" % sub[
'id']
params['static_map_url'] = self.get_static_map_url(sub, map_size)
params['static_map_width'] = map_size
params['static_map_height'] = map_size
msg_templates = self.get_message_templates()
msg.attach(
MIMEText(msg_templates['text']['confirm'].substitute(params),
'plain'))
msg.attach(
MIMEText(msg_templates['html']['confirm'].substitute(params),
'html'))
self.log(
'sending confirmation email to %s for subscription %s' %
(msg['To'], sub['id']), log.INFO)
self.send_email(msg['From'], msg['To'], msg)
# update subscription status
self.db.updateEmailSubscription(
sub['id'],
{'last_email_sent': format_datetime(datetime.now())})
def send_updates (self):
# get subscriptions in need of update
subs = self.db.getEmailSubscriptionsForUpdate ()
self.log('Sending updates to %s subscriptions ' % (len(subs)), log.INFO)
for sub in subs:
# get RSS feed
self.log('reading rss feed: %s' % (sub['rss_url']), log.INFO)
rss_complete_url = self.complete_url(sub)
feed_data = feedparser.parse(rss_complete_url)
# construct message
msg_parts = self.compose_message (sub, feed_data)
# send email
if msg_parts:
# used the last example here:
#http://docs.python.org/library/email-examples.html#email-examples
msg = MIMEMultipart('alternative')
msg['Subject'] = msg_parts['subject']
msg['From'] = '*****@*****.**'
msg['To'] = sub['email']
# The email lib has trouble with unocode, so we encode as ascii
msg.attach(MIMEText(msg_parts['text'].encode('ascii', 'replace'), 'plain'))
msg.attach(MIMEText(msg_parts['html'].encode('ascii', 'xmlcharrefreplace'), 'html'))
self.log('sending email to %s with %s new items' % (msg['To'], msg_parts['item_count']), log.INFO)
self.send_email (msg['From'], msg['To'], msg)
# update subscription status
self.db.updateEmailSubscription (sub['id'],
{'last_email_sent': format_datetime(datetime.now()), 'last_item_updated': msg_parts['last_item_updated']})
def process_item(self, task_id):
feed = self.db.getRssFeeds(task_id)
self.log('processsing rss feed %s (%s)' % (feed['id'], feed['url']), log.INFO)
# parse feed
feed_data = feedparser.parse( feed['url'] )
self.log('reading %s feed items' % (len(feed_data['items'])), log.INFO)
# update last read time
self.db.updateRssFeedLastRead (task_id)
# For each item in feed
for item in feed_data['items']:
# if this item has not already been processed
if self.db.rssFeedItemExists(item['id']):
self.log('%s - feed item already exists - skipping' % (item['id']), log.INFO)
continue
# store the full item
l=ItemLoader (RssFeedItem())
l.add_value ('item_id', item['id'])
l.add_value ('content', psycopg2.Binary(pickle.dumps(item)))
#l.add_value ('content', pickle.dumps(item))
l.add_value ('feed_id', task_id)
yield l.load_item()
#feed_entry_id = uuid.uuid5(uuid.NAMESPACE_URL, str(item ['id']))
feed_entry_id = self.db.uuid5_str(name=str(item ['id']))
l=ItemLoader (FeedEntry())
l.add_value ('id', feed_entry_id)
l.add_value ('title', item['title'])
# l.add_value ('updated', format_datetime(item['updated_parsed']))
l.add_value ('incident_datetime', format_datetime(item['updated_parsed']))
if 'content' in item:
for c in item['content']:
l.add_value ('content', c['value'])
elif 'summary' in item:
l.add_value ('content', item['summary'])
embedded_fields = self.extractContentFields (l.get_output_value('content'))
pt = embedded_fields.get('location') or item.get('georss_point')
# print "em: '%s' geo: '%s'" % (embedded_fields.get('location'), item.get('georss_point'))
if not pt:
self.log('%s - No georeference found' % (item['id']), log.WARNING)
continue
pt = re.split ("[, ]+", pt)
l.add_value ('lat', pt[0])
l.add_value ('lng', pt[1])
l.add_value ('kml_url', embedded_fields.get('kml') or '')
for link in item['links']:
if link['rel'] == 'alternate':
l.add_value ('link', link['href'])
l.add_value ('source_id', feed['source_id'])
yield l.load_item()
if 'tags' in item:
for t in item['tags']:
l=ItemLoader (FeedEntryTag())
l.add_value ('feed_entry_id', feed_entry_id)
l.add_value ('tag', t['term'])
l.add_value ('comment', t['label'])
yield l.load_item()
l=ItemLoader (FeedEntryTag())
l.add_value ('feed_entry_id', feed_entry_id)
l.add_value ('tag', feed['tag'])
yield l.load_item()
# update task status
self.item_completed (task_id)
def process_item(self, task_id):
feed = self.db.getRssFeeds(task_id)
self.log('processsing rss feed %s (%s)' % (feed['id'], feed['url']),
log.INFO)
# parse feed
feed_data = feedparser.parse(feed['url'])
self.log('reading %s feed items' % (len(feed_data['items'])), log.INFO)
# update last read time
self.db.updateRssFeedLastRead(task_id)
# For each item in feed
for item in feed_data['items']:
# if this item has not already been processed
if self.db.rssFeedItemExists(item['id']):
self.log(
'%s - feed item already exists - skipping' % (item['id']),
log.INFO)
continue
# store the full item
l = ItemLoader(RssFeedItem())
l.add_value('item_id', item['id'])
l.add_value('content', psycopg2.Binary(pickle.dumps(item)))
#l.add_value ('content', pickle.dumps(item))
l.add_value('feed_id', task_id)
yield l.load_item()
#feed_entry_id = uuid.uuid5(uuid.NAMESPACE_URL, str(item ['id']))
feed_entry_id = self.db.uuid5_str(name=str(item['id']))
l = ItemLoader(FeedEntry())
l.add_value('id', feed_entry_id)
l.add_value('title', item['title'])
# l.add_value ('updated', format_datetime(item['updated_parsed']))
l.add_value('incident_datetime',
format_datetime(item['updated_parsed']))
if 'content' in item:
for c in item['content']:
l.add_value('content', c['value'])
elif 'summary' in item:
l.add_value('content', item['summary'])
embedded_fields = self.extractContentFields(
l.get_output_value('content'))
pt = embedded_fields.get('location') or item.get('georss_point')
# print "em: '%s' geo: '%s'" % (embedded_fields.get('location'), item.get('georss_point'))
if not pt:
self.log('%s - No georeference found' % (item['id']),
log.WARNING)
continue
pt = re.split("[, ]+", pt)
l.add_value('lat', pt[0])
l.add_value('lng', pt[1])
l.add_value('kml_url', embedded_fields.get('kml') or '')
for link in item['links']:
if link['rel'] == 'alternate':
l.add_value('link', link['href'])
l.add_value('source_id', feed['source_id'])
yield l.load_item()
if 'tags' in item:
for t in item['tags']:
l = ItemLoader(FeedEntryTag())
l.add_value('feed_entry_id', feed_entry_id)
l.add_value('tag', t['term'])
l.add_value('comment', t['label'])
yield l.load_item()
l = ItemLoader(FeedEntryTag())
l.add_value('feed_entry_id', feed_entry_id)
l.add_value('tag', feed['tag'])
yield l.load_item()
# update task status
self.item_completed(task_id)
