def make_tweet(self, note):
max_length = 136
content = note.content
# Convert or remove HTML.
content = strip_tags(content)
# If the note fits in a tweet, post it as-is.
# Tweets of notes this short don't need to link to their source.
if len(content.splitlines()) == 1 and len(content) <= max_length:
return content
# Make a short URL pointing to the original note.
# Replace the middle of the private note URL to get a usable public link.
private_url = note.id
short_username = self.pump_username.split('@')[0]
public_url = private_url.replace('/api/note/',
'/' + short_username + '/note/')
# Make room for the ellipsis and URL at the end of the tweet.
# Note that all links on Twitter are shortened via t.co. Therefore, all links
# are 22 or 23 characters, depending on whether they use HTTP or HTTPS.
cropped_length = max_length - 23 - 2
# Keep only the first line.
content = content.splitlines()[0]
content = content[:cropped_length]
content = content.rstrip()
tweet = content + u'… ' + public_url
return tweet
def make_tweet(self, note):
max_length = 136
content = note.content
# Convert or remove HTML.
content = strip_tags(content)
# If the note fits in a tweet, post it as-is.
# Tweets of notes this short don't need to link to their source.
if len(content.splitlines()) == 1 and len(content) <= max_length:
return content
# Make a short URL pointing to the original note.
# Replace the middle of the private note URL to get a usable public link.
private_url = note.id
short_username = self.pump_username.split('@')[0]
public_url = private_url.replace('/api/note/', '/' + short_username + '/note/')
# Make room for the ellipsis and URL at the end of the tweet.
# Note that all links on Twitter are shortened via t.co. Therefore, all links
# are 22 or 23 characters, depending on whether they use HTTP or HTTPS.
cropped_length = max_length - 23 - 2
# Keep only the first line.
content = content.splitlines()[0]
content = content[:cropped_length]
content = content.rstrip()
tweet = content + u'… ' + public_url
return tweet
def make_tweet(note):
max_length = 136
content = note.content
# Convert or remove HTML.
content = strip_tags(content)
# If the note fits in a tweet, post it as-is.
# Tweets of notes this short don't need to link to their source.
if len(content.splitlines()) == 1 and len(content) <= max_length:
return content
# Make a short URL pointing to the original note.
private_url = note.id
public_url = private_url.replace('/api/note/', '/dper/note/')
short_url = shorten(public_url)
# Make room for the ellipsis and URL at the end of the tweet.
cropped_length = max_length - len(short_url) - 2
# Keep only the first line.
content = content.splitlines()[0]
content = content[:cropped_length]
content = content.rstrip()
tweet = content + u'… ' + short_url
return tweet
def make_tweet(self, note):
max_length = 136
content = note.content
# Convert or remove HTML.
content = strip_tags(content)
# If the note fits in a tweet, post it as-is.
# Tweets of notes this short don't need to link to their source.
if len(content.splitlines()) == 1 and len(content) <= max_length:
return content
# Make a short URL pointing to the original note.
# Replace the middle of the private note URL to get a usable public link.
private_url = note.id
short_username = self.pump_username.split('@')[0]
public_url = private_url.replace('/api/note/',
'/' + short_username + '/note/')
short_url = shorten(public_url)
# Make room for the ellipsis and URL at the end of the tweet.
cropped_length = max_length - len(short_url) - 2
# Keep only the first line.
content = content.splitlines()[0]
content = content[:cropped_length]
content = content.rstrip()
tweet = content + u'… ' + short_url
return tweet
def load():
with open('content.json','r') as json_file:
letters = json.load(json_file)
# stripping html
for l in letters:
l['Text'] = strip_tags(l['Text'])[2::]
l['Text'] = l['Text'].replace('\r\n','\n')
l['Text'] = l['Text'].replace('\n\n','!x')
l['Text'] = l['Text'].replace('\n',' ')
l['Text'] = l['Text'].replace('!x','\n')
l['Text'] = l['Text'].replace('\xad','')
l['LetterDate'] = datetime.strptime(l['LetterDate'],"%Y-%m-%dT%H:%M:%S")
l['LetterHeading'] = l['LetterDate'].strftime("%A d. %d. %b %Y").capitalize() + ' - ' + l['Place']
if(l['Location']):
l['Location'] = ','.join(l['Location'].split(',')[0:2])
return letters
def send_notification(self,
event,
alert,
template_name,
sender,
recipients,
recipients_cc=[],
recipients_bcc=[],
context={}):
all_recipients = recipients + recipients_cc + recipients_bcc
self.log.info(
"Start trying to send notification to %s with event=%s of alert %s"
% (str(all_recipients), event, alert))
mail_template = self.get_email_template(template_name)
if mail_template != False:
self.log.debug(
"Found template file (%s). Ready to send notification." %
json.dumps(mail_template))
# Parse html template with django
try:
# Parse body as django template
template = self.env.get_template(
mail_template['template_file'])
content = template.render(context)
#self.log.debug("Parsed message body. Context was: %s" % (json.dumps(context)))
text_content = strip_tags(content)
# Parse subject as django template
subject_template = Template(mail_template['subject'])
subject = subject_template.render(context)
self.log.debug("Parsed message subject: %s" % subject)
# Prepare message
self.log.debug("Preparing SMTP message...")
smtpRecipients = []
msg = MIMEMultipart('alternative')
msg['Subject'] = subject
msg['From'] = sender
msg['Date'] = formatdate(localtime=True)
#msg.preamble = text_content
if len(recipients) > 0:
smtpRecipients = smtpRecipients + recipients
msg['To'] = COMMASPACE.join(recipients)
if len(recipients_cc) > 0:
smtpRecipients = smtpRecipients + recipients_cc
msg['CC'] = COMMASPACE.join(recipients_cc)
if len(recipients_bcc) > 0:
smtpRecipients = smtpRecipients + recipients_bcc
msg['BCC'] = COMMASPACE.join(recipients_bcc)
# Add message body
msg.attach(MIMEText(text_content, 'plain'))
if mail_template['content_type'] == "html":
msg.attach(MIMEText(content, 'html'))
# Add attachments
if 'attachments' in mail_template and mail_template[
'attachments'] != None and mail_template[
'attachments'] != "":
attachment_list = mail_template['attachments'].split(" ")
self.log.debug(
"Have to add attachments to this notification. Attachment list: %s"
% json.dumps(attachment_list))
for attachment in attachment_list or []:
local_file = os.path.join(
os.environ.get('SPLUNK_HOME'), "etc", "apps",
"alert_manager", "local", "templates",
"attachments", attachment)
default_file = os.path.join(
os.environ.get('SPLUNK_HOME'), "etc", "apps",
"alert_manager", "default", "templates",
"attachments", attachment)
attachment_file = None
if os.path.isfile(local_file):
attachment_file = local_file
self.log.debug(
"%s exists in local, using this one..." %
attachment)
else:
self.log.debug(
"%s not found in local folder, checking if there's one in default..."
% attachment)
if os.path.isfile(default_file):
attachment_file = default_file
self.log.debug(
"%s exists in default, using this one..." %
attachment)
else:
self.log.warn(
"%s doesn't exist, won't add it to the message."
% attachment)
if attachment_file != None:
ctype, encoding = mimetypes.guess_type(
attachment_file)
if ctype is None or encoding is not None:
ctype = "application/octet-stream"
maintype, subtype = ctype.split("/", 1)
if maintype == "text":
fp = open(attachment_file)
# Note: we should handle calculating the charset
msgAttachment = MIMEText(fp.read(),
_subtype=subtype)
fp.close()
elif maintype == "image":
fp = open(attachment_file, "rb")
msgAttachment = MIMEImage(fp.read(),
_subtype=subtype)
fp.close()
elif maintype == "audio":
fp = open(attachment_file, "rb")
msgAttachment = MIMEAudio(fp.read(),
_subtype=subtype)
fp.close()
else:
fp = open(attachment_file, "rb")
msgAttachment = MIMEBase(maintype, subtype)
msgAttachment.set_payload(fp.read())
fp.close()
encoders.encode_base64(msgAttachment)
msgAttachment.add_header(
"Content-ID",
"<" + basename(attachment_file) + "@splunk>")
msgAttachment.add_header(
"Content-Disposition",
"attachment",
filename=basename(attachment_file))
msg.attach(msgAttachment)
#self.log.debug("Mail message: %s" % msg.as_string())
#self.log.debug("Settings: %s " % json.dumps(self.settings))
self.log.debug("smtpRecipients: %s type: %s" %
(smtpRecipients, type(smtpRecipients)))
self.log.info("Connecting to mailserver=%s ssl=%s tls=%s" %
(self.settings["MAIL_SERVER"],
self.settings["EMAIL_USE_SSL"],
self.settings["EMAIL_USE_TLS"]))
if not self.settings["EMAIL_USE_SSL"]:
s = smtplib.SMTP(self.settings["MAIL_SERVER"])
else:
s = smtplib.SMTP_SSL(self.settings["MAIL_SERVER"])
if self.settings["EMAIL_USE_TLS"]:
s.starttls()
if len(self.settings["EMAIL_HOST_USER"]) > 0:
s.login(self.settings["EMAIL_HOST_USER"],
self.settings["EMAIL_HOST_PASSWORD"])
self.log.info("Sending emails....")
s.sendmail(sender, smtpRecipients, msg.as_string())
s.quit()
self.log.info("Notifications sent successfully")
#except TemplateSyntaxError, e:
# self.log.error("Unable to parse template %s. Error: %s. Continuing without sending notification..." % (mail_template['template_file'], e))
#except smtplib.SMTPServerDisconnected, e:
# self.log.error("SMTP server disconnected the connection. Error: %s" % e)
except socket.error, e:
self.log.error(
"Wasn't able to connect to mailserver. Reason: %s" % e)
#except TemplateDoesNotExist, e:
# self.log.error("Template %s not found in %s nor %s. Continuing without sending notification..." % (mail_template['template_file'], local_dir, default_dir))
except Exception as e:
self.log.error(
"Unable to send notification. Continuing without sending notification. Unexpected Error: %s"
% (traceback.format_exc()))
def send_notification(self, event, alert, template_name, sender, recipients, recipients_cc=[], recipients_bcc=[], context = {}):
all_recipients = recipients + recipients_cc + recipients_bcc
self.log.info("Start trying to send notification to %s with event=%s of alert %s" % (str(all_recipients), event, alert))
mail_template = self.get_email_template(template_name)
if mail_template != False:
self.log.debug("Found template file (%s). Ready to send notification." % json.dumps(mail_template))
# Parse html template with django
try:
# Parse body as django template
template = self.env.get_template(mail_template['template_file'])
content = template.render(context)
#self.log.debug("Parsed message body. Context was: %s" % (json.dumps(context)))
text_content = strip_tags(content)
# Parse subject as django template
subject_template = Template(source=mail_template['subject'], variable_start_string='$', variable_end_string='$')
subject = subject_template.render(context)
self.log.debug("Parsed message subject: %s" % subject)
# Prepare message
self.log.debug("Preparing SMTP message...")
msgRoot = MIMEMultipart('related')
msgRoot['Subject'] = subject
msgRoot['From'] = sender
msgRoot['Date'] = formatdate(localtime = True)
smtpRecipients = []
msg = MIMEMultipart('alternative')
msgRoot.attach(msg)
#msg.preamble = text_content
if len(recipients) > 0:
smtpRecipients = smtpRecipients + recipients
msgRoot['To'] = COMMASPACE.join(recipients)
if len(recipients_cc) > 0:
smtpRecipients = smtpRecipients + recipients_cc
msgRoot['CC'] = COMMASPACE.join(recipients_cc)
if len(recipients_bcc) > 0:
smtpRecipients = smtpRecipients + recipients_bcc
msgRoot['BCC'] = COMMASPACE.join(recipients_bcc)
# Add message body
if mail_template['content_type'] == "html":
msg.attach(MIMEText(content, 'html'))
else:
msg.attach(MIMEText(text_content, 'plain'))
# Add attachments
if 'attachments' in mail_template and mail_template['attachments'] != None and mail_template['attachments'] != "":
attachment_list = mail_template['attachments'].split(" ")
self.log.debug("Have to add attachments to this notification. Attachment list: %s" % json.dumps(attachment_list))
for attachment in attachment_list or []:
local_file = os.path.join(os.environ.get('SPLUNK_HOME'), "etc", "apps", "alert_manager", "local", "templates", "attachments", attachment)
default_file = os.path.join(os.environ.get('SPLUNK_HOME'), "etc", "apps", "alert_manager", "default", "templates", "attachments", attachment)
attachment_file = None
if os.path.isfile(local_file):
attachment_file = local_file
self.log.debug("%s exists in local, using this one..." % attachment)
else:
self.log.debug("%s not found in local folder, checking if there's one in default..." % attachment)
if os.path.isfile(default_file):
attachment_file = default_file
self.log.debug("%s exists in default, using this one..." % attachment)
else:
self.log.warn("%s doesn't exist, won't add it to the message." % attachment)
if attachment_file != None:
ctype, encoding = mimetypes.guess_type(attachment_file)
if ctype is None or encoding is not None:
ctype = "application/octet-stream"
maintype, subtype = ctype.split("/", 1)
msgAttachment = None
if maintype == "text":
try:
fp = open(attachment_file)
# Note: we should handle calculating the charset
msgAttachment = MIMEText(fp.read(), _subtype=subtype)
finally:
fp.close()
elif maintype == "image":
try:
fp = open(attachment_file, "rb")
msgAttachment = MIMEImage(fp.read(), _subtype=subtype)
finally:
fp.close()
elif maintype == "audio":
try:
fp = open(attachment_file, "rb")
msgAttachment = MIMEAudio(fp.read(), _subtype=subtype)
finally:
fp.close()
else:
try:
fp = open(attachment_file, "rb")
msgAttachment = MIMEBase(maintype, subtype)
msgAttachment.set_payload(fp.read())
encoders.encode_base64(msgAttachment)
finally:
fp.close()
if msgAttachment != None:
msgAttachment.add_header("Content-ID", "<" + basename(attachment_file) + "@splunk>")
msgAttachment.add_header("Content-Disposition", "attachment", filename=basename(attachment_file))
msgRoot.attach(msgAttachment)
#self.log.debug("Mail message: %s" % msg.as_string())
#self.log.debug("Settings: %s " % json.dumps(self.settings))
self.log.debug("smtpRecipients: %s type: %s" % (smtpRecipients, type(smtpRecipients)))
self.log.info("Connecting to mailserver=%s ssl=%s tls=%s" % (self.settings["MAIL_SERVER"], self.settings["EMAIL_USE_SSL"], self.settings["EMAIL_USE_TLS"]))
if not self.settings["EMAIL_USE_SSL"]:
s = smtplib.SMTP(self.settings["MAIL_SERVER"])
else:
s = smtplib.SMTP_SSL(self.settings["MAIL_SERVER"])
if self.settings["EMAIL_USE_TLS"]:
s.starttls()
if len(self.settings["EMAIL_HOST_USER"]) > 0:
s.login(str(self.settings["EMAIL_HOST_USER"]), str(self.settings["EMAIL_HOST_PASSWORD"]))
self.log.info("Sending emails....")
s.sendmail(sender, smtpRecipients, msgRoot.as_string().encode('utf-8'))
s.quit()
self.log.info("Notifications sent successfully")
#except TemplateSyntaxError, e:
# self.log.error("Unable to parse template %s. Error: %s. Continuing without sending notification..." % (mail_template['template_file'], e))
#except smtplib.SMTPServerDisconnected, e:
# self.log.error("SMTP server disconnected the connection. Error: %s" % e)
except socket.error, e:
self.log.error("Wasn't able to connect to mailserver. Reason: %s" % e)
#except TemplateDoesNotExist, e:
# self.log.error("Template %s not found in %s nor %s. Continuing without sending notification..." % (mail_template['template_file'], local_dir, default_dir))
except Exception as e:
self.log.error("Unable to send notification. Continuing without sending notification. Unexpected Error: %s" % (traceback.format_exc()))
def insert_feeditem_tags(feeditem_id):
#get feed_item from our db, get title and description of item and use it as text to be wikified
conn = mdb.connect(dbc.host, dbc.user, dbc.passwrd, dbc.db, charset="utf8")
c = conn.cursor()
c.execute("""SELECT title, description, link FROM feeditem
WHERE id = %s """, int(feeditem_id),)
row = c.fetchone()
item_title = strip_tags(row[0])
item_desc = strip_tags(row[1])
item_url = row[2]
textToWikify = filterTitleDescription(item_title, item_desc)
textToWikify = urllib.quote_plus(textToWikify.encode('utf-8')) # replace special chars in string using %xx escape
'''
#remove stop words for this feed from text to be wikfied, usually feed specific stopwords are those that are repeated a lot in some feed, like the name of a news agency in its own articles
for stopList in stopLists:
if(item_url.lower() in stopList[0].lower()): #if the base url of feed is in feed item url, ie this item belongs to that feed
print('textToWikify before:')
print(textToWikify)
textToWikify = re.sub(stopList[1], '', textToWikify)
print('textToWikify after:')
print(textToWikify)
'''
#call wikify
url = params.baseUrl + 'wikify?minProbability=' + params.minProbability + '&repeatMode=' + params.repeatMode + '&source=' + textToWikify
#if not isinstance(url,unicode):
url = url.encode('utf8')
dom = None
try:
dom = wikify(url)
except:
pass
if dom: # if wikifier successfully wikified the source and no timeout occurred
topics = dom.getElementsByTagName('DetectedTopic')
for topic in topics:
topic_title = topic.getAttribute('title')
topic_id_miner = topic.getAttribute('id') #the id that wikipediaminer associates with each wikipedia topic
topic_weight = topic.getAttribute('weight')
#store this tag(ie topic) in the tags table if not already there
#first check to see if tag already exists in tag table
try:
c.execute("""SELECT * FROM tag WHERE title = %s""", (topic_title,))
tag = c.fetchone()
# if this tag doesn't exist already insert it
if(tag == None):
c.execute('SELECT id FROM tag WHERE title = %s', (topic_title,))
if len(c.fetchall()) == 0:
c.execute(
"""INSERT INTO tag (title,
url,
count,
date,
id_miner)
VALUES (%s, %s, %s, %s, %s)""",
(topic_title,
"http://en.wikipedia.org/wiki/" + topic_title, #url is null for now since wikifier doesn't return a url but it can be inferred from title
1,
datetime.datetime.now(),
topic_id_miner,)
)
else:
print("tag already exists: " + topic_title)
#soheilTODO increment count indicating this tag has been seen again
#do another select to get the tag_id to be associated with the feed_item
c.execute("""SELECT * FROM tag WHERE title = %s""", (topic_title,))
tag = c.fetchone()
tag_id = tag[0]
#associate the tag with the feeditem
c.execute(
"""INSERT INTO feeditem_tag (feeditem_id,
tag_id,
date,
weight_miner)
VALUES (%s, %s, %s, %s)""",
(feeditem_id,
tag_id,
datetime.datetime.now(),
topic_weight,)
)
except RuntimeError as error:
print error
conn.commit()
c.close()
conn.close()
conn.commit()
c.close()
conn.close()
return topics
else:
print "WARNING: Wikifier timed out on feeditem with id %s. Feeditem was deleted." % feeditem_id
conn = mdb.connect(dbc.host, dbc.user, dbc.passwrd, dbc.db, charset="utf8")
c = conn.cursor()
try:
c.execute('DELETE FROM feeditem WHERE id = %s', (feeditem_id,))
conn.commit()
c.close()
conn.close()
except RuntimeError as error:
print error
conn.commit()
c.close()
conn.close()
