Userprofile.fio).filter(Userprofile.system_id == 4):
threads = [
str(r[0]) for r in dbconn.query(Threads.id_global).filter(
Threads.performer_id == profile_id).filter(
Threads.id_local == 1).filter(
Threads.status == Status.text).filter(
Status.id == 1).filter(Threads.title_id == Tasktype.id)
.filter(Tasktype.system_id == 0).filter(
Threads.creation_date == now.strftime('%Y-%m-%d')).filter(
Threads.creation_time < delta1.strftime('%H:%M:%S')).filter(
Threads.creation_time > delta.strftime('%H:%M:%S'))
]
if threads:
do = True
mess += """
<tr><td>%s</td><td>%s</td></tr>
""" % (fio.encode('utf-8'), ','.join(threads))
mess += """
</table>
"""
if do:
msg.attach(MIMEText(mess, 'html', _charset='utf-8'))
smtp = smtplib.SMTP()
smtp.connect()
smtp.sendmail('*****@*****.**', em, msg.as_string())
smtp.sendmail('*****@*****.**', em2, msg.as_string())
# smtp.sendmail('*****@*****.**', em3, msg.as_string())
# smtp.sendmail('*****@*****.**', em4, msg.as_string())
smtp.sendmail('*****@*****.**', '*****@*****.**', msg.as_string())
smtp.close()
def add_message(self, message):
body = MIMEText(message, 'plain', 'utf-8')
self.msg.attach(body)
body = '''Что-то пошло не так и вам поставили плохую оценку :(
-----------------------------------------------
Время звонка : %s
Номер абонента : %s
Номер оператора : %s
Имя оператора : %s
Оценка : %s
Очередь : %s
-----------------------------------------------
Файл с записью разговора прикреплен к сообщению.
''' % (date, in_num, op_num, op_name, evaluation, queue)
msg.attach(MIMEText(body, 'plain', 'utf-8'))
attach_name = uid + '.mp3'
filename = '/var/spool/asterisk/monitor/' + uid + '.wav.mp3'
attachment = open(filename, "rb")
part = MIMEBase('application', 'octet-stream')
part.set_payload((attachment).read())
encoders.encode_base64(part)
part.add_header('Content-Disposition',
"attachment; filename= %s" % attach_name)
msg.attach(part)
server = smtplib.SMTP('mail.*****.net', 25)
text = msg.as_string()
def setUp(self):
self._msg = MIMEText('hello there')
Header(name, 'utf-8').encode(), \
addr.encode('utf-8') if isinstance(addr, unicode) else addr)) #.. if .. else .. 结构
from_addr = '*****@*****.**'
password = '******'
to_addr = '*****@*****.**'
smtp_server = 'smtp.163.com'
msg = MIMEMultipart() #MIMEMultipart对象代表邮件本身
msg['From'] = _format_addr(u'Python爱好者 <%s>' % from_addr)
msg['To'] = _format_addr(u'管理员 <%s>' % to_addr)
msg['Subject'] = Header(u'来自SMTP的问候……', 'utf-8').encode()
#邮件正文是MIMEText:
msg.attach(MIMEText('send with file...', 'plain', 'utf-8'))
'''
把一个图片嵌入到邮件正文中
'''
#大部分邮件服务商都会自动屏蔽带有外链的图片，因为不知道这些链接是否指向恶意网站。要把图片嵌入到邮件正文中，我们只需按照发送附件的方式，
#先把邮件作为附件添加进去，然后，在HTML中通过引用src="cid:0"就可以把附件作为图片嵌入了。如果有多个图片，给它们依次编号，然后引用不同的cid:x即可
msg.attach(
MIMEText(
'<html><body><h1>Hello</h1>' + '<p><img src="cid:0"></p>' +
'</body></html>', 'html', 'utf-8'))
#添加附件就是加上一个MIMEBase，从本地读取一个图片:
with open('test.jpg', 'rb') as f:
#设置附件的MIME和文件名，这里是jpg类型:
mime = MIMEBase('image', 'jpg', filename='test.jpg')
#加上必要的头信息:
def test_encode_noop(self):
eq = self.assertEqual
msg = MIMEText('hello world', _encoder=Encoders.encode_noop)
eq(msg.get_payload(), 'hello world\n')
eq(msg['content-transfer-encoding'], None)
def test_encode_base64(self):
eq = self.assertEqual
msg = MIMEText('hello world', _encoder=Encoders.encode_base64)
eq(msg.get_payload(), 'aGVsbG8gd29ybGQK\n')
eq(msg['content-transfer-encoding'], 'base64')
def welcome_email(user_email, wallet, uuid):
if user_email is not None:
msg = MIMEMultipart('alternative')
msg['From'] = email_from
msg['To'] = user_email
msg['Subject'] = "Welcome to Omniwallet"
text = (
'Welcome To Omniwallet!\n'
'\n'
'Thank you for creating a new wallet and choosing to join the exciting world of cryptocurrency.\n'
'While we know you are eager to get started, this email contains important information about your new Omniwallet.\n'
'So please take a moment to read through it completely.\n'
'\n'
'Your Wallet Login Details'
'This is your Wallet ID: ' + str(uuid) + '\n'
'Never share your Wallet ID or Password with anyone. Be sure to keep them safe and stored separately for security.\n\n'
'This is your unique Login Link: https://' + str(email_domain) +
'/login/' + str(uuid) + '\n'
'Bookmark this, you can use it to login directly to your Omniwallet with your password.\n'
'\n'
'Omniwallet NEVER STORES Your Password.\n'
'This means you need your password to access your wallet and the private keys within.\n'
'If you lose or forget your password there is nothing we can do to recover it.\n'
'This may result in the loss of funds in any wallet addresses which have not been Backed Up!\n\n'
'Please, Please, Please Keep Your Password Safe!\n'
'\n'
'Important Information On Backing Up Your Wallet'
'If you lose or forget your password the only thing you can use to recover your funds is a Wallet Backup.\n'
'You should create a new backup any time you make a change to your wallet (add/remove an address).\n'
'Remove the old backup only after you have confirmed the contents of the new backup are complete.\n'
'On the "My Addresses" page you can export a backup of your wallet under the "Wallet Options" button.\n'
'This backup file contains every address and private key (if known) for the addresses in your wallet at the time of backup.\n'
'Store your backup file in a secure place. Anyone with access to this file has access to your funds.'
'\n'
'Thank you for taking the time to read this introduction. \n'
'This as well as many more questions/answers are available on our FAQ Page: https://'
+ str(email_domain) + '/about/faq \n'
'If you have any questions please feel free to reach out to us using the information on our Contact Us page: https://'
+ str(email_domain) + '/about/contact \n'
'\n'
'Sincerely, \n The Omniwallet Team')
html = (
'<html><head></head><body style="background-color:rgba(234, 235, 236, 0.43);">'
'<img src="https://' + str(email_domain) +
'/assets/img/logo.png"><h1><font color="#034F92">Welcome To Omniwallet!</font></h1>'
'<p>'
'Thank you for creating a new wallet and choosing to join the exciting world of cryptocurrency.<br>'
'While we know you are eager to get started, this email contains important information about your new Omniwallet.<br>'
'So please take a moment to read through it completely.<br>'
'</p>'
'<h2><font color="#034F92">Your Wallet Login Details</font></h2>'
'<p>'
'This is your <b>Wallet ID:</b> ' + str(uuid) + '<br>'
'Never share your Wallet ID or Password with anyone. Be sure to keep them safe and stored separately for security.<br><br>'
'This is your unique <b>Login Link:</b> <a href="https://' +
str(email_domain) + '/login/' + str(uuid) + '">https://' +
str(email_domain) + '/login/' + str(uuid) + '</a><br>'
'Bookmark this, you can use it to login directly to your Omniwallet with your password.<br>'
'</p><p>'
'<strong>Omniwallet Never Stores Your Password.</strong><br>'
'This means you need your password to access your wallet and the private keys within.<br>'
'If you lose or forget your password there is nothing we can do to recover it.<br>'
'This may result in the loss of funds in any wallet addresses which have not been Backed Up!<br><br>'
'<strong>Please, Please, Please Keep Your Password Safe!</strong><br>'
'</p>'
'<h2><font color="#034F92">Important Information On Backing Up Your Wallet</font></h2>'
'<p>'
'If you lose or forget your password the only thing you can use to recover your funds is a Wallet Backup.<br>'
'You should create a new backup any time you make a change to your wallet (add/remove an address).<br>'
'Remove the old backup only after you have confirmed the contents of the new backup are complete.<br>'
'On the "My Addresses" page you can export a backup of your wallet under the "Wallet Options" button.<br>'
'This backup file contains every address and private key (if known) for the addresses in your wallet at the time of backup.<br>'
'<strong>Store your backup file in a secure place. Anyone with access to this file has access to your funds.</strong>'
'</p><p>'
'Thank you for taking the time to read this introduction. <br>'
'This as well as many more questions/answers are available on our <a href="https://'
+ str(email_domain) + '/about/faq">FAQ</a> page.<br>'
'If you have any questions please feel free to reach out to us using the information on our <a href="https://'
+ str(email_domain) + '/about/contact">Contact Us</a> page.<br>'
'</p><p>'
'Sincerely, <br><i> The Omniwallet Team</i>'
'</p></body></html>')
part1 = MIMEText(text, 'plain')
part2 = MIMEText(html, 'html')
msg.attach(part1)
msg.attach(part2)
if config.WELCOMECID is not None:
msg.add_header('X-Mailgun-Campaign-Id', config.WELCOMECID)
#wfile = MIMEBase('application', 'octet-stream')
#wfile.set_payload(wallet)
#Encoders.encode_base64(wfile)
#wfile.add_header('Content-Disposition', 'attachment', filename=uuid+'.json')
#msg.attach(wfile)
smtp = smtplib.SMTP(config.SMTPDOMAIN, config.SMTPPORT)
if config.SMTPUSER is not None and config.SMTPPASS is not None:
if config.SMTPSTARTTLS:
smtp.starttls()
smtp.login(config.SMTPUSER, config.SMTPPASS)
smtp.sendmail(email_from, user_email, msg.as_string())
smtp.close()
def _send_raw_email(self):
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[
self.current]['starttime']
# self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['stat'] = STAT_STARTED
obj = self.queue[self.current]
# create MIME message
msg = MIMEMultipart()
msg['Subject'] = self.queue[self.current]['subject']
msg['Message-Id'] = make_msgid('calibre-web')
msg['Date'] = formatdate(localtime=True)
text = self.queue[self.current]['text']
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
if obj['attachment']:
result = get_attachment(obj['filepath'], obj['attachment'])
if result:
msg.attach(result)
else:
self._handleError(u"Attachment not found")
return
msg['From'] = obj['settings']["mail_from"]
msg['To'] = obj['recipent']
use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
try:
# convert MIME message to string
fp = StringIO()
gen = Generator(fp, mangle_from_=False)
gen.flatten(msg)
msg = fp.getvalue()
# send email
timeout = 600 # set timeout to 5mins
org_stderr = sys.stderr
sys.stderr = StderrLogger()
if use_ssl == 2:
self.asyncSMTP = email_SSL(obj['settings']["mail_server"],
obj['settings']["mail_port"],
timeout)
else:
self.asyncSMTP = email(obj['settings']["mail_server"],
obj['settings']["mail_port"], timeout)
# link to logginglevel
if web.ub.config.config_log_level != logging.DEBUG:
self.asyncSMTP.set_debuglevel(0)
else:
self.asyncSMTP.set_debuglevel(1)
if use_ssl == 1:
self.asyncSMTP.starttls()
if obj['settings']["mail_password"]:
self.asyncSMTP.login(str(obj['settings']["mail_login"]),
str(obj['settings']["mail_password"]))
self.asyncSMTP.sendmail(obj['settings']["mail_from"],
obj['recipent'], msg)
self.asyncSMTP.quit()
self._handleSuccess()
sys.stderr = org_stderr
except (MemoryError) as e:
self._handleError(u'Error sending email: ' + e.message)
return None
except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
if hasattr(e, "smtp_error"):
text = e.smtp_error.decode('utf-8').replace("\n", '. ')
elif hasattr(e, "message"):
text = e.message
else:
text = ''
self._handleError(u'Error sending email: ' + text)
return None
except (socket.error) as e:
self._handleError(u'Error sending email: ' + e.strerror)
return None
for val in dic[key]:
if search_for in val: return key
return None
faddr = raw_input(
"What gmail account would you like to send from: ") + '@gmail.com'
pword = getpass.getpass(prompt="Password please: ")
taddr = raw_input("Who would you like to send an email to: ")
msg = MIMEMultipart()
msg['From'] = faddr
msg['To'] = taddr
msg['Subject'] = raw_input("What is your subject line: ")
msg.attach(
MIMEText(raw_input("What would you like the body to contain?\n"), 'plain'))
if search(BOOL, raw_input("Do you want an attachment? ").upper()) == 'TRUE':
fname = raw_input("What is the file name and extension: ")
attachment = open("/nfs/2016/s/sjones/Downloads/wat.jpeg", "rb")
aprt = MIMEBase('application', 'octet-stream')
aprt.set_payload((attachment).read())
encoders.encode_base64(aprt)
aprt.add_header('Content-Disposition', "attachment; filename= %s" % fname)
msg.attach(aprt)
else:
print '\n'
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(faddr, pword)
notification_emails_string += email
else:
notification_emails_string += ", "
notification_emails_string += email
print "Found " + str(len(missing_users)) + " missing users " + str(
missing_users
) + " that need added to SaM LDAP, sending notifications to " + str(
notification_emails) + "."
notification_email_body += "Missing user(s) from group " + group + ":" + str(
missing_users) + "\n"
print ""
# Send the notification email if needed
if notification_email_body != "":
msg = MIMEMultipart()
msg["From"] = "*****@*****.**"
msg["To"] = notification_emails_string
msg["Subject"] = "Missing user(s) in SaM LDAP"
msg.attach(MIMEText(notification_email_body))
smtp = smtplib.SMTP("localhost")
smtp.sendmail("*****@*****.**", notification_emails, msg.as_string())
smtp.quit()
ad_ldap_obj.unbind_s()
sam_ldap_obj.unbind_s()
]
#send_to = ["*****@*****.**"]
msg['In-Reply-To'] = hn_message_id
msg['References'] = hn_message_id
msg['From'] = "*****@*****.**"
msg['reply-to'] = COMMASPACE.join(reply_to)
msg['To'] = COMMASPACE.join(send_to)
msg['Date'] = formatdate(localtime=True)
msg['Subject'] = title
msg['Message-ID'] = email.Utils.make_msgid()
messageText = "Dear all,\n"
messageText = messageText + "\n"
messageText = messageText + "This batch has been assigned.\n"
messageText = messageText + "\n"
messageText = messageText + "RelVal Batch Manager"
try:
msg.attach(MIMEText(messageText))
smtpObj = smtplib.SMTP()
smtpObj.connect()
smtpObj.sendmail("*****@*****.**", send_to,
msg.as_string())
smtpObj.close()
except Exception as e:
print "Error: unable to send email: %s" % (str(e))
time.sleep(100)
def _send_raw_email(self):
self.doLock.acquire()
index = self.current
self.doLock.release()
self.queue[index]['starttime'] = datetime.now()
self.UIqueue[index]['formStarttime'] = self.queue[index]['starttime']
self.UIqueue[index]['stat'] = STAT_STARTED
obj = self.queue[index]
# create MIME message
msg = MIMEMultipart()
msg['Subject'] = self.queue[index]['subject']
msg['Message-Id'] = make_msgid('ebooks-web-app')
msg['Date'] = formatdate(localtime=True)
text = self.queue[index]['text']
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
if obj['attachment']:
result = get_attachment(obj['filepath'], obj['attachment'])
if result:
msg.attach(result)
else:
self._handleError(u"Attachment not found")
return
msg['From'] = obj['settings']["mail_from"]
msg['To'] = obj['recipent']
use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
try:
# convert MIME message to string
fp = StringIO()
gen = Generator(fp, mangle_from_=False)
gen.flatten(msg)
msg = fp.getvalue()
# send email
timeout = 600 # set timeout to 5mins
# redirect output to logfile on python2 pn python3 debugoutput is caught with overwritten
# _print_debug function
if sys.version_info < (3, 0):
org_smtpstderr = smtplib.stderr
smtplib.stderr = logger.StderrLogger('worker.smtp')
if use_ssl == 2:
self.asyncSMTP = email_SSL(obj['settings']["mail_server"],
obj['settings']["mail_port"],
timeout=timeout)
else:
self.asyncSMTP = email(obj['settings']["mail_server"],
obj['settings']["mail_port"],
timeout=timeout)
# link to logginglevel
if logger.is_debug_enabled():
self.asyncSMTP.set_debuglevel(1)
if use_ssl == 1:
self.asyncSMTP.starttls()
if obj['settings']["mail_password"]:
self.asyncSMTP.login(str(obj['settings']["mail_login"]),
str(obj['settings']["mail_password"]))
self.asyncSMTP.sendmail(obj['settings']["mail_from"],
obj['recipent'], msg)
self.asyncSMTP.quit()
self._handleSuccess()
if sys.version_info < (3, 0):
smtplib.stderr = org_smtpstderr
except (MemoryError) as e:
self._handleError(u'Error sending email: ' + e.message)
return None
except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
if hasattr(e, "smtp_error"):
text = e.smtp_error.decode('utf-8').replace("\n", '. ')
elif hasattr(e, "message"):
text = e.message
else:
text = ''
self._handleError(u'Error sending email: ' + text)
return None
except (socket.error) as e:
self._handleError(u'Error sending email: ' + e.strerror)
return None
'http://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE',
line) # Change Accesion number to Accession hyperlink.
out.write(line)
# Email
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
fromaddr = "*****@*****.**" # sender email
toaddr = "" # receiver email
msg = MIMEMultipart()
msg['From'] = fromaddr
msg['To'] = toaddr
msg['Subject'] = "GEO Data Sets - Supported platforms"
# To send send txt file in email body.
f1 = (open("email_data.txt", 'rU'))
geo = MIMEText(f1.read(), 'plain')
f1.close()
msg.attach(geo)
# Convert to string.
import smtplib
server = smtplib.SMTP('smtp.gmail.com', 587)
server.ehlo()
server.starttls()
server.ehlo()
server.login("geo.weekly.analysis", "password")
text = msg.as_string()
server.sendmail(fromaddr, toaddr, text)
def mail_results():
# Mails .txt result to predefined address
status = dict()
status["valid_entry_len"] = EmailEntry.query.filter_by(
validity=True).count()
status["spam_entry_len"] = EmailEntry.query.filter_by(spam=True).count()
with open("result.txt", "wb") as f:
for valid_entry in EmailEntry.query.filter_by(validity=True):
f.write("{0}\r\n".format(valid_entry.get_address()))
with open("spam.txt", "wb") as f:
for valid_entry in EmailEntry.query.filter_by(spam=True):
f.write("{0}\r\n".format(valid_entry.get_address()))
# "From" address built upon AWS instance's public DNS
from_address = "bot@localhost"
to_address = request.form["address"]
status["address"] = to_address
# Preparing message
msg = MIMEMultipart()
msg['From'] = from_address
msg['To'] = to_address
msg['Subject'] = "Email addresses verification result"
body = "Check attachment."
msg.attach(MIMEText(body, 'plain'))
# And first attachment
attachment = open("result.txt", "rb")
part = MIMEBase('application', 'octet-stream')
part.set_payload(attachment.read())
encoders.encode_base64(part)
part.add_header('Content-Disposition', "attachment; filename=result.txt")
msg.attach(part)
# And spam attachment
attachment = open("spam.txt", "rb")
part = MIMEBase('application', 'octet-stream')
part.set_payload(attachment.read())
encoders.encode_base64(part)
part.add_header('Content-Disposition', "attachment; filename=spam.txt")
msg.attach(part)
try:
# And sending it
from smtplib import SMTP
server = SMTP('localhost')
text = msg.as_string()
server.sendmail(from_address, to_address, text)
server.quit()
status["success"] = True
except Exception as e:
status["success"] = False
print e
return jsonify(status)
def alert_smtp(alert, metric, context):
"""
Called by :func:`~trigger_alert` and sends an alert via smtp to the
recipients that are configured for the metric.
"""
LOCAL_DEBUG = False
logger = logging.getLogger(skyline_app_logger)
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - sending smtp alert')
logger.info('debug :: alert_smtp - Memory usage at start: %s (kb)' %
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
# FULL_DURATION to hours so that analyzer surfaces the relevant timeseries data
# in the graph
full_duration_in_hours = int(settings.FULL_DURATION) / 3600
# @added 20161229 - Feature #1830: Ionosphere alerts
# Added Ionosphere variables
base_name = str(metric[1]).replace(settings.FULL_NAMESPACE, '', 1)
if settings.IONOSPHERE_ENABLED:
timeseries_dir = base_name.replace('.', '/')
training_data_dir = '%s/%s/%s' % (settings.IONOSPHERE_DATA_FOLDER,
str(int(metric[2])), timeseries_dir)
graphite_image_file = '%s/%s.%s.graphite.%sh.png' % (
training_data_dir, base_name, skyline_app,
str(int(full_duration_in_hours)))
json_file = '%s/%s.%s.redis.%sh.json' % (
training_data_dir, base_name, skyline_app,
str(int(full_duration_in_hours)))
training_data_redis_image = '%s/%s.%s.redis.plot.%sh.png' % (
training_data_dir, base_name, skyline_app,
str(int(full_duration_in_hours)))
# For backwards compatibility
if '@' in alert[1]:
sender = settings.ALERT_SENDER
recipient = alert[1]
else:
sender = settings.SMTP_OPTS['sender']
# @modified 20160806 - Added default_recipient
try:
recipients = settings.SMTP_OPTS['recipients'][alert[0]]
use_default_recipient = False
except:
use_default_recipient = True
if use_default_recipient:
try:
recipients = settings.SMTP_OPTS['default_recipient']
logger.info(
'alert_smtp - using default_recipient as no recipients are configured for %s'
% str(alert[0]))
except:
logger.error(
'error :: alert_smtp - no known recipient for %s' %
str(alert[0]))
return False
# Backwards compatibility
if type(recipients) is str:
recipients = [recipients]
# @modified 20161229 - Feature #1830: Ionosphere alerts
# Ionosphere alerts
unencoded_graph_title = 'Skyline %s - ALERT at %s hours - %s' % (
context, str(int(full_duration_in_hours)), str(metric[0]))
# @modified 20170603 - Feature #2034: analyse_derivatives
# Added deriative functions to convert the values of metrics strictly
# increasing monotonically to their deriative products in alert graphs and
# specify it in the graph_title
known_derivative_metric = False
try:
REDIS_ALERTER_CONN = redis.StrictRedis(
unix_socket_path=settings.REDIS_SOCKET_PATH)
except:
logger.error('error :: alert_smtp - redis connection failed')
try:
derivative_metrics = list(
REDIS_ALERTER_CONN.smembers('derivative_metrics'))
except:
derivative_metrics = []
redis_metric_name = '%s%s' % (settings.FULL_NAMESPACE, str(base_name))
if redis_metric_name in derivative_metrics:
known_derivative_metric = True
if known_derivative_metric:
try:
non_derivative_monotonic_metrics = settings.NON_DERIVATIVE_MONOTONIC_METRICS
except:
non_derivative_monotonic_metrics = []
skip_derivative = in_list(redis_metric_name,
non_derivative_monotonic_metrics)
if skip_derivative:
known_derivative_metric = False
if known_derivative_metric:
unencoded_graph_title = 'Skyline %s - ALERT at %s hours - derivative graph - %s' % (
context, str(int(full_duration_in_hours)), str(metric[0]))
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - unencoded_graph_title: %s' %
unencoded_graph_title)
graph_title_string = quote(unencoded_graph_title, safe='')
graph_title = '&title=%s' % graph_title_string
graphite_port = '80'
if settings.GRAPHITE_PORT != '':
graphite_port = str(settings.GRAPHITE_PORT)
link = '%s://%s:%s/render/?from=-%shours&target=cactiStyle(%s)%s%s&colorList=orange' % (
settings.GRAPHITE_PROTOCOL, settings.GRAPHITE_HOST, graphite_port,
str(int(full_duration_in_hours)), metric[1],
settings.GRAPHITE_GRAPH_SETTINGS, graph_title)
# @added 20170603 - Feature #2034: analyse_derivatives
if known_derivative_metric:
link = '%s://%s:%s/render/?from=-%shours&target=cactiStyle(nonNegativeDerivative(%s))%s%s&colorList=orange' % (
settings.GRAPHITE_PROTOCOL, settings.GRAPHITE_HOST, graphite_port,
str(int(full_duration_in_hours)), metric[1],
settings.GRAPHITE_GRAPH_SETTINGS, graph_title)
content_id = metric[1]
image_data = None
if settings.SMTP_OPTS.get('embed-images'):
# @added 20161229 - Feature #1830: Ionosphere alerts
# Use existing data if files exist
if os.path.isfile(graphite_image_file):
try:
with open(graphite_image_file, 'r') as f:
image_data = f.read()
logger.info('alert_smtp - using existing png - %s' %
graphite_image_file)
except:
logger.error(traceback.format_exc())
logger.error(
'error :: alert_smtp - failed to read image data from existing png - %s'
% graphite_image_file)
logger.error('error :: alert_smtp - %s' % str(link))
image_data = None
if image_data is None:
try:
# @modified 20170913 - Task #2160: Test skyline with bandit
# Added nosec to exclude from bandit tests
image_data = urllib2.urlopen(link).read() # nosec
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - image data OK')
except urllib2.URLError:
logger.error(traceback.format_exc())
logger.error('error :: alert_smtp - failed to get image graph')
logger.error('error :: alert_smtp - %s' % str(link))
image_data = None
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - image data None')
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage after image_data: %s (kb)' %
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
# If we failed to get the image or if it was explicitly disabled,
# use the image URL instead of the content.
if image_data is None:
img_tag = '<img src="%s"/>' % link
else:
img_tag = '<img src="cid:%s"/>' % content_id
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - img_tag: %s' % img_tag)
if settings.IONOSPHERE_ENABLED:
# Create Ionosphere Graphite image
# @modified 20161229 - Feature #1830: Ionosphere alerts
# Only write the data to the file if it does not exist
if not os.path.isfile(graphite_image_file):
try:
write_data_to_file(skyline_app, graphite_image_file, 'w',
image_data)
logger.info('added %s Ionosphere Graphite image :: %s' %
(skyline_app, graphite_image_file))
except:
logger.info(traceback.format_exc())
logger.error(
'error :: failed to add %s Ionosphere Graphite image' %
(skyline_app, graphite_image_file))
else:
logger.info(
'%s Ionosphere Graphite image already exists :: %s' %
(skyline_app, graphite_image_file))
redis_image_data = None
try:
plot_redis_data = settings.PLOT_REDIS_DATA
except:
plot_redis_data = False
if settings.SMTP_OPTS.get('embed-images') and plot_redis_data:
# Create graph from Redis data
redis_metric_key = '%s%s' % (settings.FULL_NAMESPACE, metric[1])
try:
raw_series = REDIS_ALERTER_CONN.get(redis_metric_key)
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - raw_series: %s' % 'OK')
except:
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - raw_series: %s' % 'FAIL')
try:
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage before get Redis timeseries data: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
unpacker = Unpacker(use_list=True)
unpacker.feed(raw_series)
timeseries_x = [float(item[0]) for item in unpacker]
unpacker = Unpacker(use_list=True)
unpacker.feed(raw_series)
timeseries_y = [item[1] for item in unpacker]
unpacker = Unpacker(use_list=False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage after get Redis timeseries data: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
except:
logger.error('error :: alert_smtp - unpack timeseries failed')
timeseries = None
if settings.IONOSPHERE_ENABLED and timeseries:
'''
.. todo: this is possibly to be used to allow the user to submit the
FULL_DURATION duration data set for the features profile to be
created against IF it is a Mirage metric. This would allow for
additional granularity in Mirage metrics, thereby maintaining
their seasonality, but allow user and Skyline to analyze the
anomaly at a FULL_DURATION resolution as well. Not sure how to
code that in Ionosphere context yet but could just be additonal
flag in the Ionosphere record. In the Ionosphere frontend, the
user would be given an option to either create the features
profile on the Mirage timeseries or the redis FULL_DURATION
timeseries. It is a little complicated, but doable.
# @modified 20161229 - Feature #1828: ionosphere - mirage Redis data features
However that ^^ is UNDESIRABLE in the Mirage/Ionosphere context
at the moment. Ionosphere must only profile SECOND_ORDER_RESOLUTION_HOURS
currently so as to not pollute the seasonality aspect of Mirage
'''
# Create Ionosphere redis timeseries json if is does not exist
# @modified 20161229 - Feature #1830: Ionosphere alerts
# Only write the data to the file if it does not exist and replace
# the timeseries object if a json file exists
# @added 20170920 - Bug #2168: Strange Redis derivative graph
using_original_redis_json = False
if not os.path.isfile(json_file):
timeseries_json = str(timeseries).replace('[', '(').replace(
']', ')')
try:
write_data_to_file(skyline_app, json_file, 'w',
timeseries_json)
logger.info(
'added %s Ionosphere Redis data timeseries json file :: %s'
% (skyline_app, json_file))
except:
logger.info(traceback.format_exc())
logger.error(
'error :: failed to add %s Ionosphere Redis data timeseries json file'
% (skyline_app, json_file))
else:
# Replace the timeseries object
logger.info(
'%s Ionosphere Redis data timeseries json file already exists, using :: %s'
% (skyline_app, json_file))
anomaly_json = json_file
try:
# Read the timeseries json file
with open(anomaly_json, 'r') as f:
raw_timeseries = f.read()
timeseries_array_str = str(raw_timeseries).replace(
'(', '[').replace(')', ']')
timeseries = literal_eval(timeseries_array_str)
logger.info(
'%s Redis timeseries replaced with timeseries from :: %s'
% (skyline_app, anomaly_json))
timeseries_x = [float(item[0]) for item in timeseries]
timeseries_y = [item[1] for item in timeseries]
# @added 20170920 - Bug #2168: Strange Redis derivative graph
# This already has nonNegativeDerivative applied to it
using_original_redis_json = True
except:
logger.error(traceback.format_exc())
logger.error(
'error :: %s failed to read timeseries data from %s' %
(skyline_app, anomaly_json))
timeseries = None
# @added 20170603 - Feature #2034: analyse_derivatives
if known_derivative_metric:
# @added 20170920 - Bug #2168: Strange Redis derivative graph
# If this is the Mirage Redis json it already has
# nonNegativeDerivative applied to it
if not using_original_redis_json:
logger.info('alert_smtp - nonNegativeDerivative being applied')
try:
derivative_timeseries = nonNegativeDerivative(timeseries)
timeseries = derivative_timeseries
# @added 20170920 - Bug #2168: Strange Redis derivative graph
logger.info('alert_smtp - nonNegativeDerivative applied')
except:
logger.error(
'error :: alert_smtp - nonNegativeDerivative failed')
else:
logger.info(
'alert_smtp - nonNegativeDerivative not being applied, as it will have been applied in the original json'
)
# @added 21070726 - Bug #2068: Analyzer smtp alert error on Redis plot with derivative metrics
# If the nonNegativeDerivative has been calculated we need to reset the
# x and y as nonNegativeDerivative has to discard the first value as it
# has no delta for it so the timeseries is 1 item less.
timeseries_x = [float(item[0]) for item in timeseries]
timeseries_y = [item[1] for item in timeseries]
pd_series_values = None
if timeseries:
try:
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage before pd.Series: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
values = pd.Series([x[1] for x in timeseries])
# Because the truth value of a Series is ambiguous
pd_series_values = True
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage after pd.Series: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
except:
logger.error(
'error :: alert_smtp - pandas value series on timeseries failed'
)
if pd_series_values:
try:
array_median = np.median(values)
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - values median: %s' %
str(array_median))
array_amax = np.amax(values)
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - array_amax: %s' %
str(array_amax))
array_amin = np.amin(values)
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - array_amin: %s' %
str(array_amin))
mean = values.mean()
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - mean: %s' % str(mean))
stdDev = values.std()
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - stdDev: %s' %
str(stdDev))
sigma3 = 3 * stdDev
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - sigma3: %s' %
str(sigma3))
# sigma3_series = [sigma3] * len(values)
sigma3_upper_bound = mean + sigma3
try:
sigma3_lower_bound = mean - sigma3
except:
sigma3_lower_bound = 0
sigma3_upper_series = [sigma3_upper_bound] * len(values)
sigma3_lower_series = [sigma3_lower_bound] * len(values)
amax_series = [array_amax] * len(values)
amin_series = [array_amin] * len(values)
mean_series = [mean] * len(values)
except:
logger.error(
'error :: alert_smtp - numpy ops on series failed')
mean_series = None
if mean_series:
graph_title = 'Skyline %s - ALERT - at %s hours - Redis data\n%s - anomalous value: %s' % (
context, str(
int(full_duration_in_hours)), metric[1], str(metric[0]))
# @added 20170603 - Feature #2034: analyse_derivatives
if known_derivative_metric:
graph_title = 'Skyline %s - ALERT - at %s hours - Redis data (derivative graph)\n%s - anomalous value: %s' % (
context, str(int(full_duration_in_hours)), metric[1],
str(metric[0]))
# @modified 20160814 - Bug #1558: Memory leak in Analyzer
# I think the buf is causing a memory leak, trying a file
# if python_version == 3:
# buf = io.StringIO()
# else:
# buf = io.BytesIO()
buf = '%s/%s.%s.%s.png' % (settings.SKYLINE_TMP_DIR, skyline_app,
str(int(metric[2])), metric[1])
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage before plot Redis data: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
# Too big
# rcParams['figure.figsize'] = 12, 6
rcParams['figure.figsize'] = 8, 4
try:
# fig = plt.figure()
fig = plt.figure(frameon=False)
ax = fig.add_subplot(111)
ax.set_title(graph_title, fontsize='small')
ax.set_axis_bgcolor('black')
try:
datetimes = [
dt.datetime.utcfromtimestamp(ts) for ts in timeseries_x
]
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - datetimes: %s' %
'OK')
except:
logger.error('error :: alert_smtp - datetimes: %s' %
'FAIL')
plt.xticks(rotation=0, horizontalalignment='center')
xfmt = DateFormatter('%a %H:%M')
plt.gca().xaxis.set_major_formatter(xfmt)
ax.xaxis.set_major_formatter(xfmt)
ax.plot(datetimes,
timeseries_y,
color='orange',
lw=0.6,
zorder=3)
ax.tick_params(axis='both', labelsize='xx-small')
max_value_label = 'max - %s' % str(array_amax)
ax.plot(datetimes,
amax_series,
lw=1,
label=max_value_label,
color='m',
ls='--',
zorder=4)
min_value_label = 'min - %s' % str(array_amin)
ax.plot(datetimes,
amin_series,
lw=1,
label=min_value_label,
color='b',
ls='--',
zorder=4)
mean_value_label = 'mean - %s' % str(mean)
ax.plot(datetimes,
mean_series,
lw=1.5,
label=mean_value_label,
color='g',
ls='--',
zorder=4)
sigma3_text = (r'3$\sigma$')
# sigma3_label = '%s - %s' % (str(sigma3_text), str(sigma3))
sigma3_upper_label = '%s upper - %s' % (
str(sigma3_text), str(sigma3_upper_bound))
ax.plot(datetimes,
sigma3_upper_series,
lw=1,
label=sigma3_upper_label,
color='r',
ls='solid',
zorder=4)
if sigma3_lower_bound > 0:
sigma3_lower_label = '%s lower - %s' % (
str(sigma3_text), str(sigma3_lower_bound))
ax.plot(datetimes,
sigma3_lower_series,
lw=1,
label=sigma3_lower_label,
color='r',
ls='solid',
zorder=4)
ax.get_yaxis().get_major_formatter().set_useOffset(False)
ax.get_yaxis().get_major_formatter().set_scientific(False)
# Shrink current axis's height by 10% on the bottom
box = ax.get_position()
ax.set_position([
box.x0, box.y0 + box.height * 0.1, box.width,
box.height * 0.9
])
# Put a legend below current axis
ax.legend(loc='upper center',
bbox_to_anchor=(0.5, -0.05),
fancybox=True,
shadow=True,
ncol=4,
fontsize='x-small')
plt.rc('lines', lw=2, color='w')
plt.grid(True)
ax.grid(b=True,
which='both',
axis='both',
color='lightgray',
linestyle='solid',
alpha=0.5,
linewidth=0.6)
ax.set_axis_bgcolor('black')
rcParams['xtick.direction'] = 'out'
rcParams['ytick.direction'] = 'out'
ax.margins(y=.02, x=.03)
# tight_layout removes the legend box
# fig.tight_layout()
try:
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage before plt.savefig: %s (kb)'
%
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
plt.savefig(buf, format='png')
if settings.IONOSPHERE_ENABLED:
if not os.path.exists(training_data_dir):
mkdir_p(training_data_dir)
logger.info('created dir - %s' % training_data_dir)
if not os.path.isfile(training_data_redis_image):
try:
plt.savefig(training_data_redis_image,
format='png')
logger.info(
'alert_smtp - save Redis training data image - %s'
% (training_data_redis_image))
except:
logger.info(traceback.format_exc())
logger.error(
'error :: alert_smtp - could not save - %s'
% (training_data_redis_image))
else:
logger.info(
'alert_smtp - Redis training data image already exists - %s'
% (training_data_redis_image))
# @added 20160814 - Bug #1558: Memory leak in Analyzer
# As per http://www.mail-archive.com/[email protected]/msg13222.html
# savefig in the parent process was causing the memory leak
# the below fig.clf() and plt.close() did not resolve this
# however spawing a multiprocessing process for alert_smtp
# does solve this as issue as all memory is freed when the
# process terminates.
fig.clf()
plt.close(fig)
redis_graph_content_id = 'redis.%s' % metric[1]
redis_image_data = True
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - savefig: %s' % 'OK')
logger.info(
'debug :: alert_smtp - Memory usage after plt.savefig: %s (kb)'
%
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
except:
logger.info(traceback.format_exc())
logger.error('error :: alert_smtp - plt.savefig: %s' %
'FAIL')
except:
logger.error(traceback.format_exc())
logger.error('error :: alert_smtp - could not build plot')
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage before email: %s (kb)' %
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
if redis_image_data:
redis_img_tag = '<img src="cid:%s"/>' % redis_graph_content_id
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - redis_img_tag: %s' %
str(redis_img_tag))
else:
# @modified 20161229 - Feature #1830: Ionosphere alerts
# @modified 20170108 - Feature #1852: Ionosphere - features_profile matched graphite graphs
# Restored the previous redis_img_tag method as some smtp alerts were
# coming without a Redis graph, not all but some and for some reason,
# I am pretty certain retrospectively that it was done that way from
# testing I just wanted to try and be cleaner.
# The redis_img_tag was changed at
# https://github.com/earthgecko/skyline/commit/31bcacf3f90f0953ebed0d57260cb937e01f887c#diff-520bf2a218f65074ffead4d8184c138dR489
redis_img_tag = '<img src="%s"/>' % 'none'
# redis_img_tag = '<img src="none"/>'
# @added 20170806 - Feature #1830: Ionosphere alerts
# Show a human date in alerts
alerted_at = str(dt.datetime.utcfromtimestamp(int(metric[2])))
try:
body = '<h3><font color="#dd3023">Sky</font><font color="#6698FF">line</font><font color="black"> %s alert</font></h3><br>' % context
body += '<font color="black">metric: <b>%s</b></font><br>' % metric[1]
body += '<font color="black">Anomalous value: %s</font><br>' % str(
metric[0])
body += '<font color="black">Anomaly timestamp: %s</font><br>' % str(
int(metric[2]))
# @added 20170806 - Feature #1830: Ionosphere alerts
# Show a human date in alerts
body += '<font color="black">Anomalous at: %s</font><br>' % alerted_at
body += '<font color="black">At hours: %s</font><br>' % str(
int(full_duration_in_hours))
body += '<font color="black">Next alert in: %s seconds</font><br>' % str(
alert[2])
# @added 20170603 - Feature #2034: analyse_derivatives
if known_derivative_metric:
body += '<font color="black">Derivative graph: True</font><br>'
more_body = ''
if settings.IONOSPHERE_ENABLED:
# @modified 20170823 - Bug #2142: 7bit SMTP encoding breaking long urls
# Broke body into body and more_body to workaround the 990 character
# limit per line for SMTP
more_body += '<h3><font color="#dd3023">Ionosphere :: </font><font color="#6698FF">training data</font><font color="black"></font></h3>'
ionosphere_link = '%s/ionosphere?timestamp=%s&metric=%s' % (
settings.SKYLINE_URL, str(int(metric[2])), str(metric[1]))
more_body += '<font color="black">To use this timeseries to train Skyline that this is not anomalous manage this training data at:<br>'
more_body += '<a href="%s">%s</a></font>' % (ionosphere_link,
ionosphere_link)
if redis_image_data:
more_body += '<font color="black">min: %s | max: %s | mean: %s <br>' % (
str(array_amin), str(array_amax), str(mean))
more_body += '3-sigma: %s <br>' % str(sigma3)
more_body += '3-sigma upper bound: %s | 3-sigma lower bound: %s <br></font>' % (
str(sigma3_upper_bound), str(sigma3_lower_bound))
more_body += '<h3><font color="black">Redis data at FULL_DURATION</font></h3><br>'
more_body += '<div dir="ltr">:%s<br></div>' % redis_img_tag
if image_data:
more_body += '<h3><font color="black">Graphite data at FULL_DURATION (may be aggregated)</font></h3>'
more_body += '<div dir="ltr"><a href="%s">%s</a><br></div><br>' % (
link, img_tag)
more_body += '<font color="black">Clicking on the above graph will open to the Graphite graph with current data</font><br>'
if redis_image_data:
more_body += '<font color="black">To disable the Redis data graph view, set PLOT_REDIS_DATA to False in your settings.py, if the Graphite graph is sufficient for you,<br>'
more_body += 'however do note that will remove the 3-sigma and mean value too.</font>'
more_body += '<br>'
more_body += '<div dir="ltr" align="right"><font color="#dd3023">Sky</font><font color="#6698FF">line</font><font color="black"> version :: %s</font></div><br>' % str(
skyline_version)
except:
logger.error('error :: alert_smtp - could not build body')
logger.info(traceback.format_exc())
for recipient in recipients:
try:
# @modified 20170823 - Bug #2142: 7bit SMTP encoding breaking long urls
# Broke body into body and more_body to workaround the 990 character
# limit per line for SMTP, using mixed as alternative indicates that
# the client should select one of the parts for display and ignore
# the rest (tripleee - https://stackoverflow.com/a/35115938)
# msg = MIMEMultipart('alternative')
msg = MIMEMultipart('mixed')
# @added 20170812 - Bug #2142: 7bit SMTP encoding breaking long urls
# set email charset and email encodings
cs_ = charset.Charset('utf-8')
cs_.header_encoding = charset.QP
cs_.body_encoding = charset.QP
msg.set_charset(cs_)
msg['Subject'] = '[Skyline alert] - %s ALERT - %s' % (context,
metric[1])
msg['From'] = sender
msg['To'] = recipient
msg.attach(MIMEText(body, 'html'))
# @added 20170823 - Bug #2142: 7bit SMTP encoding breaking long urls
# Broke body into body and more_body to workaround the 990 character
# limit per line for SMTP
msg.replace_header('content-transfer-encoding', 'quoted-printable')
msg.attach(MIMEText(more_body, 'html'))
if redis_image_data:
try:
# @modified 20160814 - Bug #1558: Memory leak in Analyzer
# I think the buf is causing a memory leak, trying a file
# buf.seek(0)
# msg_plot_attachment = MIMEImage(buf.read())
# msg_plot_attachment = MIMEImage(buf.read())
try:
with open(buf, 'r') as f:
plot_image_data = f.read()
try:
os.remove(buf)
except OSError:
logger.error(
'error :: alert_smtp - failed to remove file - %s'
% buf)
logger.info(traceback.format_exc())
pass
except:
logger.error('error :: failed to read plot file - %s' %
buf)
plot_image_data = None
# @added 20161124 - Branch #922: ionosphere
msg_plot_attachment = MIMEImage(plot_image_data)
msg_plot_attachment.add_header(
'Content-ID', '<%s>' % redis_graph_content_id)
msg.attach(msg_plot_attachment)
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - msg_plot_attachment - redis data done'
)
except:
logger.error('error :: alert_smtp - msg_plot_attachment')
logger.info(traceback.format_exc())
if image_data is not None:
try:
msg_attachment = MIMEImage(image_data)
msg_attachment.add_header('Content-ID',
'<%s>' % content_id)
msg.attach(msg_attachment)
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - msg_attachment - Graphite img source done'
)
except:
logger.error('error :: alert_smtp - msg_attachment')
logger.info(traceback.format_exc())
except:
logger.error('error :: alert_smtp - could not attach')
logger.info(traceback.format_exc())
s = SMTP('127.0.0.1')
try:
s.sendmail(sender, recipient, msg.as_string())
if settings.ENABLE_DEBUG or LOCAL_DEBUG:
logger.info('debug :: alert_smtp - message sent to %s OK' %
str(recipient))
except:
logger.error('error :: alert_smtp - could not send email to %s' %
str(recipient))
logger.info(traceback.format_exc())
s.quit()
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage after email: %s (kb)' %
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
if redis_image_data:
# buf.seek(0)
# buf.write('none')
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage before del redis_image_data objects: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
del raw_series
del unpacker
del timeseries[:]
del timeseries_x[:]
del timeseries_y[:]
del values
del datetimes[:]
del msg_plot_attachment
del redis_image_data
# We del all variables that are floats as they become unique objects and
# can result in what appears to be a memory leak, but is not, it is
# just the way Python handles floats
del mean
del array_amin
del array_amax
del stdDev
del sigma3
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage after del redis_image_data objects: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage before del fig object: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
# @added 20160814 - Bug #1558: Memory leak in Analyzer
# Issue #21 Memory leak in Analyzer - https://github.com/earthgecko/skyline/issues/21
# As per http://www.mail-archive.com/[email protected]/msg13222.html
fig.clf()
plt.close(fig)
del fig
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage after del fig object: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage before del other objects: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
del recipients[:]
del body
del msg
del image_data
del msg_attachment
if LOCAL_DEBUG:
logger.info(
'debug :: alert_smtp - Memory usage after del other objects: %s (kb)'
% resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
return
def encode_email_part(content, content_type):
try:
return MIMEText(content.encode('ascii'), content_type, 'ascii')
except:
return MIMEText(content.encode('utf-8'), content_type, 'utf-8')
def send(self, recipients=[]):
"""Sends the newsletter.
An optional list of dicts (keys=fullname|mail) can be passed in
for sending a newsletter out addresses != subscribers.
"""
# preparations
request = self.REQUEST
# get hold of the parent Newsletter object#
enl = self.getNewsletter()
# get sender name
sender_name = request.get("sender_name", "")
if sender_name == "":
sender_name = enl.getSenderName()
# don't use Header() with a str and a charset arg, even if it is correct
# this would generate a encoded header and mail server may not support utf-8 encoded header
from_header = Header(safe_unicode(sender_name))
# get sender e-mail
sender_email = request.get("sender_email", "")
if sender_email == "":
sender_email = enl.getSenderEmail()
from_header.append('<%s>' % safe_unicode(sender_email))
# get subject
subject = request.get("subject", "")
if subject == "":
subject = self.Title()
subject_header = Header(safe_unicode(subject))
# determine MailHost first (build-in vs. external)
deliveryServiceName = enl.getDeliveryService()
if deliveryServiceName == 'mailhost':
MailHost = getToolByName(enl, 'MailHost')
else:
MailHost = getUtility(IMailHost, name=deliveryServiceName)
log.info('Using mail delivery service "%r"' % MailHost)
send_counter = 0
send_error_counter = 0
receivers = self._send_recipients(recipients)
output_html = self._render_output_html()
rendered_newsletter = self._exchange_relative_urls(output_html)
text = rendered_newsletter['html']
text_plain = rendered_newsletter['plain']
image_urls = rendered_newsletter['images']
props = getToolByName(self, "portal_properties").site_properties
charset = props.getProperty("default_charset")
portal = getSite()
for receiver in receivers:
# create multipart mail
outer = MIMEMultipart('alternative')
if hasattr(request, "test"):
outer['To'] = Header('<%s>' % safe_unicode(receiver['email']))
fullname = receiver['fullname']
salutation = receiver['salutation']
personal_text = text.replace("[[SUBSCRIBER_SALUTATION]]", "")
personal_text_plain = text_plain.replace(
"[[SUBSCRIBER_SALUTATION]]", "")
personal_text = text.replace("[[UNSUBSCRIBE]]", "")
personal_text_plain = text_plain.replace("[[UNSUBSCRIBE]]", "")
else:
if 'uid' in receiver:
try:
unsubscribe_text = enl.getUnsubscribe_string()
except AttributeError:
unsubscribe_text = "Click here to unsubscribe"
unsubscribe_link = enl.absolute_url(
) + "/unsubscribe?subscriber=" + receiver['uid']
personal_text = text.replace(
"[[UNSUBSCRIBE]]", """<a href="%s">%s.</a>""" %
(unsubscribe_link, unsubscribe_text))
personal_text_plain = text_plain.replace(
"[[UNSUBSCRIBE]]",
"""\n%s: %s""" % (unsubscribe_text, unsubscribe_link))
else:
personal_text = text.replace("[[UNSUBSCRIBE]]", "")
personal_text_plain = text_plain.replace(
"[[UNSUBSCRIBE]]", "")
if 'salutation' in receiver:
salutation = receiver["salutation"]
else:
salutation = ''
fullname = receiver['fullname']
if not fullname:
try:
fullname = enl.getFullname_fallback()
except AttributeError:
fullname = "Sir or Madam"
outer['To'] = Header('<%s>' % safe_unicode(receiver['email']))
subscriber_salutation = safe_portal_encoding(
salutation) + ' ' + safe_portal_encoding(fullname)
personal_text = personal_text.replace("[[SUBSCRIBER_SALUTATION]]",
str(subscriber_salutation))
personal_text_plain = personal_text_plain.replace(
"[[SUBSCRIBER_SALUTATION]]", str(subscriber_salutation))
outer['From'] = from_header
outer['Subject'] = subject_header
outer.epilogue = ''
# Attach text part
text_part = MIMEText(personal_text_plain, "plain", charset)
# Attach html part with images
html_part = MIMEMultipart("related")
html_text = MIMEText(personal_text, "html", charset)
html_part.attach(html_text)
# Add images to the message
image_number = 0
reference_tool = getToolByName(self, 'reference_catalog')
for image_url in image_urls:
try:
image_url = urlparse(image_url)[2]
if 'resolveuid' in image_url:
urlparts = image_url.split('/')[1:]
uuid = urlparts.pop(0)
o = reference_tool.lookupObject(uuid)
if o and urlparts:
# get thumb
o = o.restrictedTraverse(urlparts[0])
elif "@@images" in image_url:
image_url_base, image_scale_params = image_url.split(
"@@images")
image_scale = image_scale_params.split("/")[-1]
scales = self.restrictedTraverse(
urllib.unquote(
image_url_base.strip('/') + '/@@images'))
o = scales.scale('image', scale=image_scale)
else:
o = self.restrictedTraverse(urllib.unquote(image_url))
except Exception, e:
log.error("Could not resolve the image \"%s\": %s" %
(image_url, e))
else:
if hasattr(o, "_data"): # file-based
image = MIMEImage(o._data)
elif hasattr(o, "data"):
image = MIMEImage(o.data) # zodb-based
elif hasattr(o, "GET"):
image = MIMEImage(o.GET()) # z3 resource image
else:
log.error(
"Could not get the image data from image object!")
image["Content-ID"] = "<image_%s>" % image_number
image_number += 1
# attach images only to html parts
html_part.attach(image)
outer.attach(text_part)
outer.attach(html_part)
try:
#MailHost.send(msg)
MailHost.send(outer.as_string())
log.info("Send newsletter to \"%s\"" % receiver['email'])
send_counter += 1
except AttributeError: # Plone3.3.x
MailHost.send(msg.as_string())
log.info("Send newsletter to \"%s\"" % receiver['email'])
send_counter += 1
except Exception, e:
log.info(
"Sending newsletter to \"%s\" failed, with error \"%s\"!" %
(receiver['email'], e))
send_error_counter += 1
def test_encode_8bit(self):
eq = self.assertEqual
msg = MIMEText('hello \x80 world', _encoder=Encoders.encode_7or8bit)
eq(msg.get_payload(), 'hello \x80 world\n')
eq(msg['content-transfer-encoding'], '8bit')
ir_pool = self.pool.get('ir.attachment')
for to in emailto:
msg = MIMEMultipart()
msg['Subject'] = tools.ustr(subject)
msg['To'] = to
msg['From'] = context.get('email_from', smtp_server.from_email)
if body == False:
body = ''
if smtp_server.disclaimers:
body = body + "\n" + smtp_server.disclaimers
try:
msg.attach(MIMEText(body.encode(charset) or '', _charset=charset, _subtype="html"))
except:
msg.attach(MIMEText(body or '', _charset=charset, _subtype="html"))
#add custom headers to email
for hk in headers.keys():
msg[hk] = headers[hk]
for hk in smtp_server.header_ids:
msg[hk.key] = hk.value
context_headers = context.get('headers', [])
for hk in context_headers:
msg[hk] = context_headers[hk]
# Add OpenERP Server information
def test_encode_quoted_printable(self):
eq = self.assertEqual
msg = MIMEText('hello world', _encoder=Encoders.encode_quopri)
eq(msg.get_payload(), 'hello=20world\n')
eq(msg['content-transfer-encoding'], 'quoted-printable')
def main(argv):
reportdir = str(argv[1])
outputxmlfilepath = 'C:\\MM_Results\\' + reportdir + '\\output.xml'
textdatafilepath = 'C:\\Mentoring_Minds\\mysatori-qa\\Wizard_Interface\\Automation_Testsuites\\Test.txt'
SUITENAME = 'TestDataCreation'
xmldoc = parse(outputxmlfilepath)
#Read all the test suites from output.xml file
itemlist = xmldoc.getElementsByTagName('suite')
print itemlist
starttiming = ''
endtiming = ''
for suite in range(0,len(itemlist)-1) :
#Read the test suite names
suitename= itemlist[suite].attributes['name'].value
print suitename
#Read the starttime and end time of the given suite
if(suitename == SUITENAME):
statusTgList = itemlist[suite].getElementsByTagName('status')
starttiming = statusTgList[suite].attributes['starttime'].value
print starttiming
for val in range (0,len(statusTgList)-1):
endtiming = statusTgList[val].attributes['endtime'].value
print endtiming
break
datetimeobject = datetime.strptime(starttiming,'%Y%m%d %H:%M:%S.%f')
startdate = datetimeobject.strftime('%m-%d-%y')
starttime = datetimeobject.strftime('%H:%M')
datetimeobject = datetime.strptime(endtiming,'%Y%m%d %H:%M:%S.%f')
enddate = datetimeobject.strftime('%m-%d-%y')
endtime = datetimeobject.strftime('%H:%M')
print startdate
print starttime
print enddate
print endtime
#Read the execution time from output.xml file.
timevalue1 = starttiming
timevalue2 = endtiming
print timevalue1
print timevalue2
#diffval = timevalue2 - timevalue1
#print "diffval:"+str(diffval)
#diffvalList = str(diffval).split(".")
#executiontime = diffvalList[0]
filePath = textdatafilepath
f = open(filePath,"r");
contents = f.read()
schoolDct = {}
for row in contents.split("\n"):
if len(row.strip())>0:
print row
rowvalues = row.split(" = ")
schoolDct[str(rowvalues[0])]= str(rowvalues[1])
print schoolDct
if(schoolDct['NewSchoolsAdded']!='True'):
return False
with open("mailText.txt", "w") as myfile:
print schoolDct['schoolName']
string1 = str(schoolDct['schoolName'])
if 'School' in string1 :
schoolnamelist = string1.split("School")
schoolDct['schoolName'] = schoolnamelist[0]
print schoolDct['schoolName']
URL = 'http://v4.tomo.zone/mysdb_m3_sales/_design/mys/mmloginw.html#customerid:'
myfile.write(schoolDct['schoolName']+' school setup completed at '+ endtime +' on '+ enddate +'.\n')
myfile.write('\n')
myfile.write('It can now be accessed here:\n')
myfile.write(URL+schoolDct['password']+'\n')
myfile.write('\n')
myfile.write('Zip code:'+schoolDct['zip']+'\n')
myfile.write('\n')
myfile.close()
textfile = 'mailText.txt'
fp = open(textfile, 'r')
body = fp.read()
fp.close()
if(schoolDct['Teststatus'] == 'FAIL'):
return False
msg = MIMEMultipart()
# recipients email ids
recipients = ['*****@*****.**','*****@*****.**','*****@*****.**','*****@*****.**','*****@*****.**','*****@*****.**','*****@*****.**','*****@*****.**','*****@*****.**']
# Sender email details
me = '*****@*****.**'
Password = "******"
#SMTP server details
smtpservername = 'smtp.gmail.com'
subject = 'Sandbox complete for '+schoolDct['schoolName']+' School at Zip Code ['+schoolDct['zip']+']'
msg['From'] = me
msg['To'] = COMMASPACE.join(recipients)
msg['Date'] = formatdate(localtime = True)
# Send the message via the Gmail server.
msg['Subject'] = subject
msg.attach(MIMEText(body))
smtp = smtplib.SMTP('smtp.gmail.com',587)
smtp.ehlo()
smtp.starttls()
smtp.ehlo
smtp.login(me,Password)
smtp.sendmail(me, recipients, msg.as_string().replace("\\",""))
smtp.close()
print('The email subject is "' + msg['Subject'] +'"')
def mail(to, subject, prioflag1, prioflag2, text, reply_to_address=from_address):
msg = MIMEMultipart()
msg['From'] = str(
Header(from_displayname, 'UTF-8').encode() + ' <' + from_address + '> ')
msg['To'] = to
msg['Reply-To'] = str(
Header(from_displayname, 'UTF-8').encode() + ' <' + reply_to_address + '> ')
msg['X-Priority'] = prioflag1
msg['X-MSMail-Priority'] = prioflag2
msg['Subject'] = Header(subject, 'UTF-8').encode()
body_type = MIMEText(text, "%s" % (message_flag), 'UTF-8')
msg.attach(body_type)
# now attach the file
if file_format != "":
fileMsg = email.mime.base.MIMEBase('application', '')
fileMsg.set_payload(file(file_format).read())
email.encoders.encode_base64(fileMsg)
fileMsg.add_header(
'Content-Disposition', 'attachment; filename="%s"' % os.path.basename(file_format) )
msg.attach(fileMsg)
for inline_file in inline_files:
if inline_file != "":
fileMsg = email.mime.base.MIMEBase('application', '')
fileMsg.set_payload(file(inline_file).read())
email.encoders.encode_base64(fileMsg)
fileMsg.add_header(
'Content-Disposition', 'inline; filename="%s"' % os.path.basename(inline_file) )
fileMsg.add_header( "Content-ID", "<%s>" % os.path.basename(inline_file) )
msg.attach(fileMsg)
mailServer = smtplib.SMTP(smtp, port)
io = StringIO()
msggen = Generator(io, False)
msggen.flatten(msg)
if sendmail == 0:
if email_provider == "gmail" or email_provider == "yahoo" or email_provider == "hotmail":
try:
mailServer.starttls()
except:
pass
mailServer.ehlo()
else:
mailServer.ehlo()
try:
if provideruser != "" or pwd != "":
mailServer.login(provideruser, pwd)
mailServer.sendmail(from_address, to, io.getvalue())
else:
mailServer.sendmail(from_address, to, io.getvalue())
except:
# try logging in with base64 encoding here
import base64
try:
mailServer.docmd("AUTH LOGIN", base64.b64encode(provideruser))
mailServer.docmd(base64.b64encode(pwd), "")
# except exceptions and print incorrect password
except Exception as e:
print_warning(
"It appears your password was incorrect.\nPrinting response: " + (str(e)))
return_continue()
if sendmail == 1:
mailServer.sendmail(from_address, to, io.getvalue())
def _sendMail(to, fro, subject, text, server="localhost"):
assert type(to) == list
msg = MIMEMultipart()
msg['From'] = fro
msg['To'] = COMMASPACE.join(to)
msg['Date'] = formatdate(localtime=True)
msg['Subject'] = subject
msg.attach(MIMEText(text))
print "_sendMail:msg", msg
status = False
username = get_config_param('database.ini', 'Email', 'username')
password = get_config_param('database.ini', 'Email', 'password')
port = int(get_config_param('database.ini', 'Email', 'port'))
if port == False:
print "parameter port is not set"
return False
smtp = None
try:
smtp = smtplib.SMTP(server, port)
status = True
except Exception as e:
print(e)
status = False
finally:
if status == False:
print "error in smtp connection"
return status
try:
#smtp = smtplib.SMTP(server,port)
#if not smtp:
# print "smtp is not defined"
# return
print "set debug level for smtp"
smtp.set_debuglevel(True)
# identify yourself, and get supported features
smtp.ehlo()
# start tls for security
if smtp.has_extn('STARTTLS'):
print "TLS Supported"
#starttls and check if it succeeded
if smtp.starttls()[0] == 220:
print "starttls succeeded"
#return in case starttls is supported and fails
else:
print "starttls failed"
status = False
return
smtp.ehlo() # re-identify ourselves over TLS connection
if username != False and password != False:
smtp.login(username, password)
sendStatus = smtp.sendmail(fro, to, msg.as_string())
print "sendStatus=", sendStatus
if sendStatus:
print "Receipient refused"
status = False
return
print "send email succeeded"
status = True
#except:
except Exception as e:
print(e)
print "failed to send mail"
status = False
return
finally:
if smtp is not None:
smtp.quit()
return status
def _packet_in_handler(self, ev):
# If you hit this you might want to increase
# the "miss_send_length" of your switch
# if ev.msg.msg_len < ev.msg.total_len:
# self.logger.debug("packet truncated: only %s of %s bytes",
# ev.msg.msg_len, ev.msg.total_len)
msg = ev.msg
datapath = msg.datapath
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
in_port = msg.match['in_port']
pkt = packet.Packet(msg.data)
eth = pkt.get_protocols(ethernet.ethernet)[0]
if eth.ethertype == ether_types.ETH_TYPE_LLDP:
# ignore lldp packet
return
dst = eth.dst
src = eth.src
udppkt = pkt.get_protocol(udp.udp)
if udppkt != None and udppkt.dst_port ==53:
time = datetime.datetime.now().strftime("%A, %d. %B %Y %I:%M%p")
final_arr= msg.data[14+20+8+13:msg.total_len-5]
#check if the mac is matching to any of risks
for x in child_mac:
if x == src:
for y in blocklist:
if final_arr.find(y) >= 0:
fromaddr = '*****@*****.**'
username = '******'
passwd = 'hackathon1'
email_msg = MIMEMultipart()
email_msg['From'] = fromaddr
email_msg['Subject'] = 'ZODIAC ADMIN : Blacklist ALERT ' + x
message = time + ": Blocked " + x + " from " + y
email_msg.attach(MIMEText(message))
server = smtplib.SMTP('smtp.gmail.com:587')
server.ehlo()
server.starttls()
server.ehlo()
server.login(username,passwd)
for z in email:
email_msg['To'] = z
server.sendmail(fromaddr,z,email_msg.as_string())
server.quit()
print "<font size=\"3\" color=\"red\"> " + message + "</font><br>"
return
for x in blocklist:
if final_arr.find(x) >= 0:
print time + " user mac: " + src + " browsed blocklisted url " + blocklist + "<br>"
print time + " user mac: " + src + " browsed " + final_arr + "<br>"
dpid = datapath.id
self.mac_to_port.setdefault(dpid, {})
# self.logger.info("packet in %s %s %s %s", dpid, src, dst, in_port)
# learn a mac address to avoid FLOOD next time.
self.mac_to_port[dpid][src] = in_port
if dst in self.mac_to_port[dpid]:
out_port = self.mac_to_port[dpid][dst]
else:
out_port = ofproto.OFPP_FLOOD
actions = [parser.OFPActionOutput(out_port)]
# install a flow to avoid packet_in next time
if out_port != ofproto.OFPP_FLOOD:
match = parser.OFPMatch(in_port=in_port, eth_dst=dst)
# verify if we have a valid buffer_id, if yes avoid to send both
# flow_mod & packet_out
if msg.buffer_id != ofproto.OFP_NO_BUFFER:
self.add_flow(datapath, 1, match, actions, msg.buffer_id)
return
else:
self.add_flow(datapath, 1, match, actions)
data = None
if msg.buffer_id == ofproto.OFP_NO_BUFFER:
data = msg.data
out = parser.OFPPacketOut(datapath=datapath, buffer_id=msg.buffer_id,
in_port=in_port, actions=actions, data=data)
datapath.send_msg(out)
from email.MIMEText import MIMEText
import time
import RPi.GPIO as GPIO
# Pines sensor IR
GPIO.setmode(GPIO.BOARD)
GPIO.setup(40, GPIO.IN)
# Datos de envio
SourceAdress = "*******"
DestinyAdress = "*******"
server = stmplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(SourceAdress, "*******")
msg = MIMEMultipart()
msg['From'] = SourceAdress
msg['To'] = DestinyAdress
msg['Subject'] = "Alerta de seguridad"
mensaje = "ALERTA! Se ha detectado una presencia en la camara numero 1."
msg.attach(MIMEText(mensaje, 'plain'))
Texto = msg.as_string()
if GPIO.input(40) == 1:
print "Enviando correo de alerta"
server.sendmail(SourceAdress, DestinyAdress, Texto)
time.sleep(5)
else:
print "No se detecto ninguna presencia"
'Facebook': '*****@*****.**',
'Microsoft': '*****@*****.**'
}
for key, value in company_dict.iteritems():
fromaddr = "*****@*****.**"
toaddr = value
msg = MIMEMultipart()
msg['From'] = "Imaad Mohamed Khan"
msg['To'] = toaddr
msg['Subject'] = "Application at" + " " + key
body = "Hi" + " " + key + " " + "team,\nTrust this mail finds you in the pink of your health. Who writes opening lines like these?\nBest Regards,\nImaad Mohamed Khan"
msg.attach(MIMEText(body, 'plain'))
filename = "Imaad_CV.pdf"
attachment = open("Imaad_CV.pdf", "r+")
part = MIMEBase('application', 'octet-stream')
part.set_payload((attachment).read())
encoders.encode_base64(part)
part.add_header('Content-Disposition',
"attachment; filename= %s" % filename)
msg.attach(part)
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(fromaddr, "Enter Password")
from email.mime.image import MIMEImage
from email.Utils import COMMASPACE, formatdate
from email import Encoders
filePath = "attach.jpg" #your file name
From = '*****@*****.**'
To = '*****@*****.**'
msg = MIMEMultipart()
msg['From'] = From
msg['To'] = To
msg['Date'] = formatdate(localtime=True)
msg['Subject'] = 'Automated Home'
msg.attach(MIMEText('Someone is waiting'))
try:
smtp = smtplib.SMTP('smtp.gmail.com:587')
smtp.starttls()
smtp.login('*****@*****.**', 'rajiv9230')
except:
i = 1
else:
i = 0
if i == 0:
ctype, encoding = mimetypes.guess_type(filePath)
if ctype is None or encoding is not None:
# No guess could be made, or the file is encoded (compressed), so
# use a generic bag-of-bits type.
time.sleep(fill_seconds)
if off_value == 1:
GPIO.output(pump_gpio, GPIO.HIGH)
else:
GPIO.output(pump_gpio, GPIO.LOW)
time.sleep(1)
elif switch_function == "email_fill_alert":
body += output_text(">>>>>>>> Emailing Fill Alert" + "")
fillmsg = MIMEMultipart()
fillmsg['From'] = mailfrom
fillmsg['To'] = mailto
fillmsg['Subject'] = "Fill Alert on Switch " + datestamp
body = "The fill switch has been triggered"
fillmsg.attach(MIMEText(body, 'plain'))
text = fillmsg.as_string()
mailserver.sendmail(mailfrom, mailto, text)
elif switch_function == "email_fill_emergency":
body += output_text(">>>>>>>> Emailing Fill Emergency" + "")
emergmsg = MIMEMultipart()
emergmsg['From'] = mailfrom
emergmsg['To'] = mailto
emergmsg['Subject'] = "Fill Emergency on Switch " + datestamp
body = "The emergency fill switch has been triggered"
emergmsg.attach(MIMEText(body, 'plain'))
text = emergmsg.as_string()
mailserver.sendmail(mailfrom, mailto, text)
#Get Manual Pump Jobs
body += output_text("\r\n--------Getting Manual Pump Jobs--------")
def main():
mysqlconn = MySQLdb.connect(host='dbod-cmsrv1.cern.ch',
user='******',
passwd="relval",
port=5506)
curs = mysqlconn.cursor()
curs.execute("use " + dbname + ";")
#curs.execute("lock tables batches write, batches_archive write, workflows write, workflows_archive write, datasets write, clone_reinsert_requests write")
curs.execute("select * from batches")
batches = curs.fetchall()
batches_colnames = [desc[0] for desc in curs.description]
for batch in batches:
#for name, value in zip(batches_colnames, batch):
# print name+" => "+str(value)
batch_dict = dict(zip(batches_colnames, batch))
userid = batch_dict["useridyear"] + "_" + batch_dict[
"useridmonth"] + "_" + batch_dict["useridday"] + "_" + str(
batch_dict["useridnum"]) + "_" + str(
batch_dict["batch_version_num"])
if batch_dict["status"] == "input_dsets_ready":
print " userid => " + userid
curs.execute(
"select workflow_name from workflows where useridyear = \"" +
batch_dict["useridyear"] + "\" and useridmonth = \"" +
batch_dict["useridmonth"] + "\" and useridday = \"" +
batch_dict["useridday"] + "\" and useridnum = " +
str(batch_dict["useridnum"]) + " and batch_version_num = " +
str(batch_dict["batch_version_num"]) + ";")
wfs = curs.fetchall()
#first do checks to make sure the workflows do not write into an existing dataset
for wf in wfs:
conn = httplib.HTTPSConnection(
'cmsweb.cern.ch',
cert_file=os.getenv('X509_USER_PROXY'),
key_file=os.getenv('X509_USER_PROXY'))
r1 = conn.request(
"GET", '/reqmgr/reqMgr/request?requestName=' + wf[0])
r2 = conn.getresponse()
schema = json.loads(r2.read())
for key, value in schema.items():
if type(value) is dict and key.startswith("Task"):
if ('KeepOutput' in value and value['KeepOutput']
) or 'KeepOutput' not in value:
if 'InputDataset' in value:
dset = "/" + value['InputDataset'].split(
'/'
)[1] + "/" + value[
'AcquisitionEra'] + "-" + value[
'ProcessingString'] + "-v" + str(
batch_dict["processing_version"]
) + "/*"
curs.execute(
"select * from datasets where dset_name = \""
+ dset.rstrip("*") + "\";")
dbs_dset_check = utils.getDatasets(dset)
curs_fetchall = curs.fetchall()
if len(curs_fetchall) != 0:
dsets_colnames = [
desc[0] for desc in curs.description
]
dset_dict = dict(
zip(dsets_colnames, curs_fetchall[0]))
userid_previously_inserted_dset = dset_dict[
"useridyear"] + "_" + dset_dict[
"useridmonth"] + "_" + dset_dict[
"useridday"] + "_" + str(
dset_dict["useridnum"]
) + "_" + str(dset_dict[
"batch_version_num"])
os.system(
'echo \"' + userid + "\n" + wf[0] +
"\n" +
userid_previously_inserted_dset +
"\n" + dset_dict["workflow_name"] +
"\n" + dset +
'\" | mail -s \"assignor.py error 1\" [email protected]'
)
sys.exit(1)
elif len(dbs_dset_check) != 0:
os.system(
'echo \"' + userid + "\n" + wf[0] +
"\n" + dset +
'\" | mail -s \"assignment_loop.py error 5\" [email protected]'
)
sys.exit(1)
else:
curs.execute(
"insert into datasets set dset_name=\""
+ dset.rstrip("*") +
"\", workflow_name=\"" + wf[0] +
"\", useridyear = \"" +
batch_dict["useridyear"] +
"\", useridmonth = \"" +
batch_dict["useridmonth"] +
"\", useridday = \"" +
batch_dict["useridday"] +
"\", useridnum = " +
str(batch_dict["useridnum"]) +
", batch_version_num = " +
str(batch_dict["batch_version_num"]) +
";")
elif 'PrimaryDataset' in value:
dset = "/" + value[
'PrimaryDataset'] + "/" + value[
'AcquisitionEra'] + "-" + value[
'ProcessingString'] + "-v" + str(
batch_dict["processing_version"]
) + "/*"
curs.execute(
"select * from datasets where dset_name = \""
+ dset.rstrip("*") + "\";")
curs_fetchall = curs.fetchall()
dbs_dset_check = utils.getDatasets(dset)
if len(curs_fetchall) != 0:
dsets_colnames = [
desc[0] for desc in curs.description
]
dset_dict = dict(
zip(dsets_colnames, curs_fetchall[0]))
userid_previously_inserted_dset = dset_dict[
"useridyear"] + "_" + dset_dict[
"useridmonth"] + "_" + dset_dict[
"useridday"] + "_" + str(
dset_dict["useridnum"]
) + "_" + str(dset_dict[
"batch_version_num"])
os.system(
'echo \"' + userid + "\n" + wf[0] +
"\n" +
userid_previously_inserted_dset +
"\n" + dset_dict["workflow_name"] +
"\n" + dset +
'\" | mail -s \"assignment_loop.py error 2\" [email protected]'
)
sys.exit(1)
elif len(dbs_dset_check) != 0:
os.system(
'echo \"' + userid + "\n" + wf[0] +
'\" | mail -s \"assignment_loop.py error 7\" [email protected]'
)
sys.exit(1)
else:
curs.execute(
"insert into datasets set dset_name=\""
+ dset.rstrip("*") +
"\", workflow_name=\"" + wf[0] +
"\", useridyear = " +
batch_dict["useridyear"] +
", useridmonth = " +
batch_dict["useridmonth"] +
", useridday = " +
batch_dict["useridday"] +
", useridnum = " +
str(batch_dict["useridnum"]) +
", batch_version_num = " +
str(batch_dict["batch_version_num"]) +
";")
#only assign the workflows after all of the checks are done
for wf in wfs:
conn = httplib.HTTPSConnection(
'cmsweb.cern.ch',
cert_file=os.getenv('X509_USER_PROXY'),
key_file=os.getenv('X509_USER_PROXY'))
r1 = conn.request(
"GET", '/reqmgr/reqMgr/request?requestName=' + wf[0])
r2 = conn.getresponse()
if r2.status != 200:
os.system(
'echo ' + wf[0] +
' | mail -s \"assignment_loop.py error 8\" [email protected]'
)
sys.exit(0)
schema = json.loads(r2.read())
#hack because workflows assigned to only T2_CH_CERN_T0 never get acquired
site = batch_dict["site"]
#if site == "T2_CH_CERN_T0":
# site = ["T2_CH_CERN","T2_CH_CERN_T0"]
params = assignment.make_assignment_params(
schema, site, batch_dict["processing_version"])
result = reqMgrClient.assignWorkflow("cmsweb.cern.ch", wf[0],
"relval", params)
if result != True:
os.system(
'echo ' + wf[0] +
' | mail -s \"assignment_loop.py error 4\" [email protected]'
)
sys.exit(0)
time.sleep(30)
curs.execute(
"update batches set status=\"assigned\", current_status_start_time=\""
+ datetime.datetime.now().strftime("%y:%m:%d %H:%M:%S") +
"\" where useridyear = \"" + batch_dict["useridyear"] +
"\" and useridmonth = \"" + batch_dict["useridmonth"] +
"\" and useridday = \"" + batch_dict["useridday"] +
"\" and useridnum = " + str(batch_dict["useridnum"]) +
" and batch_version_num = " +
str(batch_dict["batch_version_num"]) + ";")
mysqlconn.commit()
if batch_dict[
"hn_message_id"] != "do_not_send_an_acknowledgement_email":
msg = MIMEMultipart()
reply_to = []
send_to = [
"*****@*****.**",
"*****@*****.**"
]
#send_to = ["*****@*****.**","*****@*****.**"]
#send_to = ["*****@*****.**"]
msg['In-Reply-To'] = batch_dict["hn_message_id"]
msg['References'] = batch_dict["hn_message_id"]
msg['From'] = "*****@*****.**"
msg['reply-to'] = COMMASPACE.join(reply_to)
msg['To'] = COMMASPACE.join(send_to)
msg['Date'] = formatdate(localtime=True)
msg['Subject'] = batch_dict["announcement_title"]
msg['Message-ID'] = email.Utils.make_msgid()
messageText = "Dear all,\n"
messageText = messageText + "\n"
messageText = messageText + "This batch has been assigned.\n"
messageText = messageText + "\n"
messageText = messageText + "RelVal Batch Manager"
try:
msg.attach(MIMEText(messageText))
smtpObj = smtplib.SMTP()
smtpObj.connect()
smtpObj.sendmail("*****@*****.**", send_to,
msg.as_string())
smtpObj.close()
except Exception as e:
print "Error: unable to send email: %s" % (str(e))
