Ejemplo n.º 1
0
	def make_tweet(self, note):
		max_length = 136
		content = note.content
	
		# Convert or remove HTML.
		content = strip_tags(content)
	
		# If the note fits in a tweet, post it as-is.
		# Tweets of notes this short don't need to link to their source.
		if len(content.splitlines()) == 1 and len(content) <= max_length:
			return content
	
		# Make a short URL pointing to the original note.
		# Replace the middle of the private note URL to get a usable public link.
		private_url = note.id
		short_username = self.pump_username.split('@')[0]
		public_url = private_url.replace('/api/note/', '/' + short_username + '/note/')
	
		# Make room for the ellipsis and URL at the end of the tweet.
		# Note that all links on Twitter are shortened via t.co. Therefore, all links
		# are 22 or 23 characters, depending on whether they use HTTP or HTTPS.
		cropped_length = max_length - 23 - 2
	
		# Keep only the first line.
		content = content.splitlines()[0]
		content = content[:cropped_length]
		content = content.rstrip()

		tweet = content + u'… ' + public_url	
		return tweet
Ejemplo n.º 2
0
def make_tweet(note):
	max_length = 136
	content = note.content

	# Convert or remove HTML.
	content = strip_tags(content)

	# If the note fits in a tweet, post it as-is.
	# Tweets of notes this short don't need to link to their source.
	if len(content.splitlines()) == 1 and len(content) <= max_length:
		return content

	# Make a short URL pointing to the original note.
	private_url = note.id
	public_url = private_url.replace('/api/note/', '/dper/note/')
	short_url = shorten(public_url)

	# Make room for the ellipsis and URL at the end of the tweet.
	cropped_length = max_length - len(short_url) - 2

	# Keep only the first line.
	content = content.splitlines()[0]
	content = content[:cropped_length]
	content = content.rstrip()

	tweet = content + u'… ' + short_url
	return tweet
Ejemplo n.º 3
0
    def send_notification(self, event, alert, template_name, sender, recipients, recipients_cc=[], recipients_bcc=[], context = {}):
        all_recipients = recipients + recipients_cc + recipients_bcc
        self.log.info("Start trying to send notification to %s with event=%s of alert %s" % (str(all_recipients), event, alert))       

        mail_template = self.get_email_template(template_name)        
        if mail_template != False:
            self.log.debug("Found template file (%s). Ready to send notification." % json.dumps(mail_template))

            
            # Parse html template with django 
            try: 
                # Parse body as django template
                template = self.env.get_template(mail_template['template_file'])
                content = template.render(context)
                #self.log.debug("Parsed message body. Context was: %s" % (json.dumps(context)))

                text_content = strip_tags(content)

                # Parse subject as django template
                subject_template = Template(source=mail_template['subject'], variable_start_string='$', variable_end_string='$')
                subject = subject_template.render(context)
                self.log.debug("Parsed message subject: %s" % subject)

                # Prepare message
                self.log.debug("Preparing SMTP message...")
                msgRoot = MIMEMultipart('related')
                msgRoot['Subject']  = subject
                msgRoot['From']     = sender
                msgRoot['Date']     = formatdate(localtime = True)

                smtpRecipients = []
                msg = MIMEMultipart('alternative')
                msgRoot.attach(msg)

                #msg.preamble    = text_content

                if len(recipients) > 0:
                    smtpRecipients = smtpRecipients + recipients
                    msgRoot['To']       = COMMASPACE.join(recipients)
                if len(recipients_cc) > 0:
                    smtpRecipients = smtpRecipients + recipients_cc
                    msgRoot['CC'] = COMMASPACE.join(recipients_cc)

                if len(recipients_bcc) > 0:
                    smtpRecipients = smtpRecipients + recipients_bcc
                    msgRoot['BCC'] = COMMASPACE.join(recipients_bcc)


                # Add message body
                if mail_template['content_type'] == "html":
                    msg.attach(MIMEText(content, 'html'))
                else:
                    msg.attach(MIMEText(text_content, 'plain'))

                # Add attachments
                if 'attachments' in mail_template and mail_template['attachments'] != None and mail_template['attachments'] != "":
                    attachment_list = mail_template['attachments'].split(" ")
                    self.log.debug("Have to add attachments to this notification. Attachment list: %s" % json.dumps(attachment_list))

                    for attachment in attachment_list or []:
                        local_file = os.path.join(os.environ.get('SPLUNK_HOME'), "etc", "apps", "alert_manager", "local", "templates", "attachments", attachment)
                        default_file = os.path.join(os.environ.get('SPLUNK_HOME'), "etc", "apps", "alert_manager", "default", "templates", "attachments", attachment)

                        attachment_file = None
                        if os.path.isfile(local_file):
                            attachment_file = local_file
                            self.log.debug("%s exists in local, using this one..." % attachment)
                        else:
                            self.log.debug("%s not found in local folder, checking if there's one in default..." % attachment)
                            if os.path.isfile(default_file):
                                attachment_file = default_file
                                self.log.debug("%s exists in default, using this one..." % attachment)
                            else:
                                self.log.warn("%s doesn't exist, won't add it to the message." % attachment)

                        if attachment_file != None:
                            ctype, encoding = mimetypes.guess_type(attachment_file)
                            if ctype is None or encoding is not None:
                                ctype = "application/octet-stream"
                            maintype, subtype = ctype.split("/", 1)

                            msgAttachment = None
                            if maintype == "text":
                                try:
                                    fp = open(attachment_file)
                                    # Note: we should handle calculating the charset
                                    msgAttachment = MIMEText(fp.read(), _subtype=subtype)
                                finally:
                                    fp.close()
                            elif maintype == "image":
                                try:
                                    fp = open(attachment_file, "rb")
                                    msgAttachment = MIMEImage(fp.read(), _subtype=subtype)
                                finally:
                                    fp.close()
                            elif maintype == "audio":
                                try:
                                    fp = open(attachment_file, "rb")
                                    msgAttachment = MIMEAudio(fp.read(), _subtype=subtype)
                                finally:
                                    fp.close()
                            else:
                                try:
                                    fp = open(attachment_file, "rb")
                                    msgAttachment = MIMEBase(maintype, subtype)
                                    msgAttachment.set_payload(fp.read())
                                    encoders.encode_base64(msgAttachment)
                                finally:
                                    fp.close()
                                
                            if msgAttachment != None:
                                msgAttachment.add_header("Content-ID", "<" + basename(attachment_file) + "@splunk>")
                                msgAttachment.add_header("Content-Disposition", "attachment", filename=basename(attachment_file))
                                msgRoot.attach(msgAttachment)

                #self.log.debug("Mail message: %s" % msg.as_string())
                #self.log.debug("Settings: %s " % json.dumps(self.settings))
                self.log.debug("smtpRecipients: %s type: %s" % (smtpRecipients, type(smtpRecipients)))
                self.log.info("Connecting to mailserver=%s ssl=%s tls=%s" % (self.settings["MAIL_SERVER"], self.settings["EMAIL_USE_SSL"], self.settings["EMAIL_USE_TLS"]))
                if not self.settings["EMAIL_USE_SSL"]:
                     s = smtplib.SMTP(self.settings["MAIL_SERVER"])
                else:
                     s = smtplib.SMTP_SSL(self.settings["MAIL_SERVER"])

                if self.settings["EMAIL_USE_TLS"]:
                     s.starttls()

                if len(self.settings["EMAIL_HOST_USER"]) > 0:
                    s.login(str(self.settings["EMAIL_HOST_USER"]), str(self.settings["EMAIL_HOST_PASSWORD"]))

                self.log.info("Sending emails....")
                s.sendmail(sender, smtpRecipients, msgRoot.as_string().encode('utf-8'))
                s.quit()
                
                self.log.info("Notifications sent successfully")

            #except TemplateSyntaxError, e:
            #    self.log.error("Unable to parse template %s. Error: %s. Continuing without sending notification..." % (mail_template['template_file'], e))
            #except smtplib.SMTPServerDisconnected, e:
            #    self.log.error("SMTP server disconnected the connection. Error: %s" % e)    
            except socket.error, e:
                self.log.error("Wasn't able to connect to mailserver. Reason: %s" % e)
            #except TemplateDoesNotExist, e:
            #    self.log.error("Template %s not found in %s nor %s. Continuing without sending notification..." % (mail_template['template_file'], local_dir, default_dir))
            except Exception as e:
                self.log.error("Unable to send notification. Continuing without sending notification. Unexpected Error: %s" % (traceback.format_exc()))
Ejemplo n.º 4
0
def insert_feeditem_tags(feeditem_id):
    #get feed_item from our db, get title and description of item and use it as text to be wikified
    conn = mdb.connect(dbc.host, dbc.user, dbc.passwrd, dbc.db, charset="utf8")
    c = conn.cursor()
    c.execute("""SELECT title, description, link FROM feeditem
                      WHERE id = %s """, int(feeditem_id),)
    row = c.fetchone()
    item_title = strip_tags(row[0])
    item_desc = strip_tags(row[1])
    item_url = row[2]
    textToWikify = filterTitleDescription(item_title, item_desc)
    textToWikify = urllib.quote_plus(textToWikify.encode('utf-8'))  # replace special chars in string using %xx escape

    '''
    #remove stop words for this feed from text to be wikfied, usually feed specific stopwords are those that are repeated a lot in some feed, like the name of a news agency in its own articles
    for stopList in stopLists:
        if(item_url.lower() in stopList[0].lower()): #if the base url of feed is in feed item url, ie this item belongs to that feed
            print('textToWikify before:')
            print(textToWikify)
            textToWikify = re.sub(stopList[1], '', textToWikify)
            
            print('textToWikify after:')
            print(textToWikify)
    '''
            
            
    #call wikify
    url = params.baseUrl + 'wikify?minProbability=' + params.minProbability + '&repeatMode=' + params.repeatMode + '&source=' + textToWikify
    #if not isinstance(url,unicode):
    url = url.encode('utf8')

    dom = None
    try:
        dom = wikify(url)
    except:
        pass

    if dom: # if wikifier successfully wikified the source and no timeout occurred
        topics = dom.getElementsByTagName('DetectedTopic')
        
        for topic in topics:
            topic_title = topic.getAttribute('title')
            topic_id_miner = topic.getAttribute('id') #the id that wikipediaminer associates with each wikipedia topic
            topic_weight = topic.getAttribute('weight')
            
            #store this tag(ie topic) in the tags table if not already there
            #first check to see if tag already exists in tag table
            try: 
                c.execute("""SELECT * FROM tag WHERE title = %s""", (topic_title,))
                tag = c.fetchone()
                # if this tag doesn't exist already insert it
                if(tag == None):
                    c.execute('SELECT id FROM tag WHERE title = %s', (topic_title,))
                    if len(c.fetchall()) == 0:
                        c.execute(
                              """INSERT INTO tag (title, 
                                                  url,
                                                  count, 
                                                  date, 
                                                  id_miner)
                                 VALUES (%s, %s, %s, %s, %s)""",
                                                (topic_title,
                                                  "http://en.wikipedia.org/wiki/" + topic_title, #url is null for now since wikifier doesn't return a url but it can be inferred from title
                                                  1,
                                                  datetime.datetime.now(),
                                                  topic_id_miner,) 
                                                )
                else:
                    print("tag already exists: " + topic_title)
                    #soheilTODO increment count indicating this tag has been seen again
        
                #do another select to get the tag_id to be associated with the feed_item
                c.execute("""SELECT * FROM tag WHERE title = %s""", (topic_title,))
                tag = c.fetchone()
                tag_id = tag[0] 
                        
                #associate the tag with the feeditem
                c.execute(
                      """INSERT INTO feeditem_tag (feeditem_id,
                                                   tag_id,
                                                   date,
                                                   weight_miner)
                         VALUES (%s, %s, %s, %s)""",
                                                 (feeditem_id,
                                                   tag_id,
                                                   datetime.datetime.now(),
                                                   topic_weight,) 
                                                 )
            except RuntimeError as error:
                print error
                conn.commit()
                c.close()                
                conn.close()
    
        
        conn.commit()
        c.close()
        conn.close() 
        return topics
    else:
        print "WARNING: Wikifier timed out on feeditem with id %s. Feeditem was deleted." % feeditem_id
        conn = mdb.connect(dbc.host, dbc.user, dbc.passwrd, dbc.db, charset="utf8")
        c = conn.cursor()
        try:
            c.execute('DELETE FROM feeditem WHERE id = %s', (feeditem_id,))
            conn.commit()
            c.close()
            conn.close()
        except RuntimeError as error:
            print error
            conn.commit()
            c.close()                
            conn.close()