def make_tweet(self, note): max_length = 136 content = note.content # Convert or remove HTML. content = strip_tags(content) # If the note fits in a tweet, post it as-is. # Tweets of notes this short don't need to link to their source. if len(content.splitlines()) == 1 and len(content) <= max_length: return content # Make a short URL pointing to the original note. # Replace the middle of the private note URL to get a usable public link. private_url = note.id short_username = self.pump_username.split('@')[0] public_url = private_url.replace('/api/note/', '/' + short_username + '/note/') # Make room for the ellipsis and URL at the end of the tweet. # Note that all links on Twitter are shortened via t.co. Therefore, all links # are 22 or 23 characters, depending on whether they use HTTP or HTTPS. cropped_length = max_length - 23 - 2 # Keep only the first line. content = content.splitlines()[0] content = content[:cropped_length] content = content.rstrip() tweet = content + u'… ' + public_url return tweet
def make_tweet(note): max_length = 136 content = note.content # Convert or remove HTML. content = strip_tags(content) # If the note fits in a tweet, post it as-is. # Tweets of notes this short don't need to link to their source. if len(content.splitlines()) == 1 and len(content) <= max_length: return content # Make a short URL pointing to the original note. private_url = note.id public_url = private_url.replace('/api/note/', '/dper/note/') short_url = shorten(public_url) # Make room for the ellipsis and URL at the end of the tweet. cropped_length = max_length - len(short_url) - 2 # Keep only the first line. content = content.splitlines()[0] content = content[:cropped_length] content = content.rstrip() tweet = content + u'… ' + short_url return tweet
def make_tweet(self, note): max_length = 136 content = note.content # Convert or remove HTML. content = strip_tags(content) # If the note fits in a tweet, post it as-is. # Tweets of notes this short don't need to link to their source. if len(content.splitlines()) == 1 and len(content) <= max_length: return content # Make a short URL pointing to the original note. # Replace the middle of the private note URL to get a usable public link. private_url = note.id short_username = self.pump_username.split('@')[0] public_url = private_url.replace('/api/note/', '/' + short_username + '/note/') short_url = shorten(public_url) # Make room for the ellipsis and URL at the end of the tweet. cropped_length = max_length - len(short_url) - 2 # Keep only the first line. content = content.splitlines()[0] content = content[:cropped_length] content = content.rstrip() tweet = content + u'… ' + short_url return tweet
def load(): with open('content.json','r') as json_file: letters = json.load(json_file) # stripping html for l in letters: l['Text'] = strip_tags(l['Text'])[2::] l['Text'] = l['Text'].replace('\r\n','\n') l['Text'] = l['Text'].replace('\n\n','!x') l['Text'] = l['Text'].replace('\n',' ') l['Text'] = l['Text'].replace('!x','\n') l['Text'] = l['Text'].replace('\xad','') l['LetterDate'] = datetime.strptime(l['LetterDate'],"%Y-%m-%dT%H:%M:%S") l['LetterHeading'] = l['LetterDate'].strftime("%A d. %d. %b %Y").capitalize() + ' - ' + l['Place'] if(l['Location']): l['Location'] = ','.join(l['Location'].split(',')[0:2]) return letters
def send_notification(self, event, alert, template_name, sender, recipients, recipients_cc=[], recipients_bcc=[], context={}): all_recipients = recipients + recipients_cc + recipients_bcc self.log.info( "Start trying to send notification to %s with event=%s of alert %s" % (str(all_recipients), event, alert)) mail_template = self.get_email_template(template_name) if mail_template != False: self.log.debug( "Found template file (%s). Ready to send notification." % json.dumps(mail_template)) # Parse html template with django try: # Parse body as django template template = self.env.get_template( mail_template['template_file']) content = template.render(context) #self.log.debug("Parsed message body. Context was: %s" % (json.dumps(context))) text_content = strip_tags(content) # Parse subject as django template subject_template = Template(mail_template['subject']) subject = subject_template.render(context) self.log.debug("Parsed message subject: %s" % subject) # Prepare message self.log.debug("Preparing SMTP message...") smtpRecipients = [] msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = sender msg['Date'] = formatdate(localtime=True) #msg.preamble = text_content if len(recipients) > 0: smtpRecipients = smtpRecipients + recipients msg['To'] = COMMASPACE.join(recipients) if len(recipients_cc) > 0: smtpRecipients = smtpRecipients + recipients_cc msg['CC'] = COMMASPACE.join(recipients_cc) if len(recipients_bcc) > 0: smtpRecipients = smtpRecipients + recipients_bcc msg['BCC'] = COMMASPACE.join(recipients_bcc) # Add message body msg.attach(MIMEText(text_content, 'plain')) if mail_template['content_type'] == "html": msg.attach(MIMEText(content, 'html')) # Add attachments if 'attachments' in mail_template and mail_template[ 'attachments'] != None and mail_template[ 'attachments'] != "": attachment_list = mail_template['attachments'].split(" ") self.log.debug( "Have to add attachments to this notification. Attachment list: %s" % json.dumps(attachment_list)) for attachment in attachment_list or []: local_file = os.path.join( os.environ.get('SPLUNK_HOME'), "etc", "apps", "alert_manager", "local", "templates", "attachments", attachment) default_file = os.path.join( os.environ.get('SPLUNK_HOME'), "etc", "apps", "alert_manager", "default", "templates", "attachments", attachment) attachment_file = None if os.path.isfile(local_file): attachment_file = local_file self.log.debug( "%s exists in local, using this one..." % attachment) else: self.log.debug( "%s not found in local folder, checking if there's one in default..." % attachment) if os.path.isfile(default_file): attachment_file = default_file self.log.debug( "%s exists in default, using this one..." % attachment) else: self.log.warn( "%s doesn't exist, won't add it to the message." % attachment) if attachment_file != None: ctype, encoding = mimetypes.guess_type( attachment_file) if ctype is None or encoding is not None: ctype = "application/octet-stream" maintype, subtype = ctype.split("/", 1) if maintype == "text": fp = open(attachment_file) # Note: we should handle calculating the charset msgAttachment = MIMEText(fp.read(), _subtype=subtype) fp.close() elif maintype == "image": fp = open(attachment_file, "rb") msgAttachment = MIMEImage(fp.read(), _subtype=subtype) fp.close() elif maintype == "audio": fp = open(attachment_file, "rb") msgAttachment = MIMEAudio(fp.read(), _subtype=subtype) fp.close() else: fp = open(attachment_file, "rb") msgAttachment = MIMEBase(maintype, subtype) msgAttachment.set_payload(fp.read()) fp.close() encoders.encode_base64(msgAttachment) msgAttachment.add_header( "Content-ID", "<" + basename(attachment_file) + "@splunk>") msgAttachment.add_header( "Content-Disposition", "attachment", filename=basename(attachment_file)) msg.attach(msgAttachment) #self.log.debug("Mail message: %s" % msg.as_string()) #self.log.debug("Settings: %s " % json.dumps(self.settings)) self.log.debug("smtpRecipients: %s type: %s" % (smtpRecipients, type(smtpRecipients))) self.log.info("Connecting to mailserver=%s ssl=%s tls=%s" % (self.settings["MAIL_SERVER"], self.settings["EMAIL_USE_SSL"], self.settings["EMAIL_USE_TLS"])) if not self.settings["EMAIL_USE_SSL"]: s = smtplib.SMTP(self.settings["MAIL_SERVER"]) else: s = smtplib.SMTP_SSL(self.settings["MAIL_SERVER"]) if self.settings["EMAIL_USE_TLS"]: s.starttls() if len(self.settings["EMAIL_HOST_USER"]) > 0: s.login(self.settings["EMAIL_HOST_USER"], self.settings["EMAIL_HOST_PASSWORD"]) self.log.info("Sending emails....") s.sendmail(sender, smtpRecipients, msg.as_string()) s.quit() self.log.info("Notifications sent successfully") #except TemplateSyntaxError, e: # self.log.error("Unable to parse template %s. Error: %s. Continuing without sending notification..." % (mail_template['template_file'], e)) #except smtplib.SMTPServerDisconnected, e: # self.log.error("SMTP server disconnected the connection. Error: %s" % e) except socket.error, e: self.log.error( "Wasn't able to connect to mailserver. Reason: %s" % e) #except TemplateDoesNotExist, e: # self.log.error("Template %s not found in %s nor %s. Continuing without sending notification..." % (mail_template['template_file'], local_dir, default_dir)) except Exception as e: self.log.error( "Unable to send notification. Continuing without sending notification. Unexpected Error: %s" % (traceback.format_exc()))
def send_notification(self, event, alert, template_name, sender, recipients, recipients_cc=[], recipients_bcc=[], context = {}): all_recipients = recipients + recipients_cc + recipients_bcc self.log.info("Start trying to send notification to %s with event=%s of alert %s" % (str(all_recipients), event, alert)) mail_template = self.get_email_template(template_name) if mail_template != False: self.log.debug("Found template file (%s). Ready to send notification." % json.dumps(mail_template)) # Parse html template with django try: # Parse body as django template template = self.env.get_template(mail_template['template_file']) content = template.render(context) #self.log.debug("Parsed message body. Context was: %s" % (json.dumps(context))) text_content = strip_tags(content) # Parse subject as django template subject_template = Template(source=mail_template['subject'], variable_start_string='$', variable_end_string='$') subject = subject_template.render(context) self.log.debug("Parsed message subject: %s" % subject) # Prepare message self.log.debug("Preparing SMTP message...") msgRoot = MIMEMultipart('related') msgRoot['Subject'] = subject msgRoot['From'] = sender msgRoot['Date'] = formatdate(localtime = True) smtpRecipients = [] msg = MIMEMultipart('alternative') msgRoot.attach(msg) #msg.preamble = text_content if len(recipients) > 0: smtpRecipients = smtpRecipients + recipients msgRoot['To'] = COMMASPACE.join(recipients) if len(recipients_cc) > 0: smtpRecipients = smtpRecipients + recipients_cc msgRoot['CC'] = COMMASPACE.join(recipients_cc) if len(recipients_bcc) > 0: smtpRecipients = smtpRecipients + recipients_bcc msgRoot['BCC'] = COMMASPACE.join(recipients_bcc) # Add message body if mail_template['content_type'] == "html": msg.attach(MIMEText(content, 'html')) else: msg.attach(MIMEText(text_content, 'plain')) # Add attachments if 'attachments' in mail_template and mail_template['attachments'] != None and mail_template['attachments'] != "": attachment_list = mail_template['attachments'].split(" ") self.log.debug("Have to add attachments to this notification. Attachment list: %s" % json.dumps(attachment_list)) for attachment in attachment_list or []: local_file = os.path.join(os.environ.get('SPLUNK_HOME'), "etc", "apps", "alert_manager", "local", "templates", "attachments", attachment) default_file = os.path.join(os.environ.get('SPLUNK_HOME'), "etc", "apps", "alert_manager", "default", "templates", "attachments", attachment) attachment_file = None if os.path.isfile(local_file): attachment_file = local_file self.log.debug("%s exists in local, using this one..." % attachment) else: self.log.debug("%s not found in local folder, checking if there's one in default..." % attachment) if os.path.isfile(default_file): attachment_file = default_file self.log.debug("%s exists in default, using this one..." % attachment) else: self.log.warn("%s doesn't exist, won't add it to the message." % attachment) if attachment_file != None: ctype, encoding = mimetypes.guess_type(attachment_file) if ctype is None or encoding is not None: ctype = "application/octet-stream" maintype, subtype = ctype.split("/", 1) msgAttachment = None if maintype == "text": try: fp = open(attachment_file) # Note: we should handle calculating the charset msgAttachment = MIMEText(fp.read(), _subtype=subtype) finally: fp.close() elif maintype == "image": try: fp = open(attachment_file, "rb") msgAttachment = MIMEImage(fp.read(), _subtype=subtype) finally: fp.close() elif maintype == "audio": try: fp = open(attachment_file, "rb") msgAttachment = MIMEAudio(fp.read(), _subtype=subtype) finally: fp.close() else: try: fp = open(attachment_file, "rb") msgAttachment = MIMEBase(maintype, subtype) msgAttachment.set_payload(fp.read()) encoders.encode_base64(msgAttachment) finally: fp.close() if msgAttachment != None: msgAttachment.add_header("Content-ID", "<" + basename(attachment_file) + "@splunk>") msgAttachment.add_header("Content-Disposition", "attachment", filename=basename(attachment_file)) msgRoot.attach(msgAttachment) #self.log.debug("Mail message: %s" % msg.as_string()) #self.log.debug("Settings: %s " % json.dumps(self.settings)) self.log.debug("smtpRecipients: %s type: %s" % (smtpRecipients, type(smtpRecipients))) self.log.info("Connecting to mailserver=%s ssl=%s tls=%s" % (self.settings["MAIL_SERVER"], self.settings["EMAIL_USE_SSL"], self.settings["EMAIL_USE_TLS"])) if not self.settings["EMAIL_USE_SSL"]: s = smtplib.SMTP(self.settings["MAIL_SERVER"]) else: s = smtplib.SMTP_SSL(self.settings["MAIL_SERVER"]) if self.settings["EMAIL_USE_TLS"]: s.starttls() if len(self.settings["EMAIL_HOST_USER"]) > 0: s.login(str(self.settings["EMAIL_HOST_USER"]), str(self.settings["EMAIL_HOST_PASSWORD"])) self.log.info("Sending emails....") s.sendmail(sender, smtpRecipients, msgRoot.as_string().encode('utf-8')) s.quit() self.log.info("Notifications sent successfully") #except TemplateSyntaxError, e: # self.log.error("Unable to parse template %s. Error: %s. Continuing without sending notification..." % (mail_template['template_file'], e)) #except smtplib.SMTPServerDisconnected, e: # self.log.error("SMTP server disconnected the connection. Error: %s" % e) except socket.error, e: self.log.error("Wasn't able to connect to mailserver. Reason: %s" % e) #except TemplateDoesNotExist, e: # self.log.error("Template %s not found in %s nor %s. Continuing without sending notification..." % (mail_template['template_file'], local_dir, default_dir)) except Exception as e: self.log.error("Unable to send notification. Continuing without sending notification. Unexpected Error: %s" % (traceback.format_exc()))
def insert_feeditem_tags(feeditem_id): #get feed_item from our db, get title and description of item and use it as text to be wikified conn = mdb.connect(dbc.host, dbc.user, dbc.passwrd, dbc.db, charset="utf8") c = conn.cursor() c.execute("""SELECT title, description, link FROM feeditem WHERE id = %s """, int(feeditem_id),) row = c.fetchone() item_title = strip_tags(row[0]) item_desc = strip_tags(row[1]) item_url = row[2] textToWikify = filterTitleDescription(item_title, item_desc) textToWikify = urllib.quote_plus(textToWikify.encode('utf-8')) # replace special chars in string using %xx escape ''' #remove stop words for this feed from text to be wikfied, usually feed specific stopwords are those that are repeated a lot in some feed, like the name of a news agency in its own articles for stopList in stopLists: if(item_url.lower() in stopList[0].lower()): #if the base url of feed is in feed item url, ie this item belongs to that feed print('textToWikify before:') print(textToWikify) textToWikify = re.sub(stopList[1], '', textToWikify) print('textToWikify after:') print(textToWikify) ''' #call wikify url = params.baseUrl + 'wikify?minProbability=' + params.minProbability + '&repeatMode=' + params.repeatMode + '&source=' + textToWikify #if not isinstance(url,unicode): url = url.encode('utf8') dom = None try: dom = wikify(url) except: pass if dom: # if wikifier successfully wikified the source and no timeout occurred topics = dom.getElementsByTagName('DetectedTopic') for topic in topics: topic_title = topic.getAttribute('title') topic_id_miner = topic.getAttribute('id') #the id that wikipediaminer associates with each wikipedia topic topic_weight = topic.getAttribute('weight') #store this tag(ie topic) in the tags table if not already there #first check to see if tag already exists in tag table try: c.execute("""SELECT * FROM tag WHERE title = %s""", (topic_title,)) tag = c.fetchone() # if this tag doesn't exist already insert it if(tag == None): c.execute('SELECT id FROM tag WHERE title = %s', (topic_title,)) if len(c.fetchall()) == 0: c.execute( """INSERT INTO tag (title, url, count, date, id_miner) VALUES (%s, %s, %s, %s, %s)""", (topic_title, "http://en.wikipedia.org/wiki/" + topic_title, #url is null for now since wikifier doesn't return a url but it can be inferred from title 1, datetime.datetime.now(), topic_id_miner,) ) else: print("tag already exists: " + topic_title) #soheilTODO increment count indicating this tag has been seen again #do another select to get the tag_id to be associated with the feed_item c.execute("""SELECT * FROM tag WHERE title = %s""", (topic_title,)) tag = c.fetchone() tag_id = tag[0] #associate the tag with the feeditem c.execute( """INSERT INTO feeditem_tag (feeditem_id, tag_id, date, weight_miner) VALUES (%s, %s, %s, %s)""", (feeditem_id, tag_id, datetime.datetime.now(), topic_weight,) ) except RuntimeError as error: print error conn.commit() c.close() conn.close() conn.commit() c.close() conn.close() return topics else: print "WARNING: Wikifier timed out on feeditem with id %s. Feeditem was deleted." % feeditem_id conn = mdb.connect(dbc.host, dbc.user, dbc.passwrd, dbc.db, charset="utf8") c = conn.cursor() try: c.execute('DELETE FROM feeditem WHERE id = %s', (feeditem_id,)) conn.commit() c.close() conn.close() except RuntimeError as error: print error conn.commit() c.close() conn.close()