def notify_us_of_long_outages(hours=720):
outages = get_long_running_outages(hours)
msg = []
msg.append("The following outages have been running for longer than %d hours:\n" % hours)
for outage in outages:
try:
msg.append(
"outage %d, duration %s, server %d (%s), customer %d (%s)"
% (
outage.id,
outage.getDuration(approximate=True),
outage.server.id,
outage.server.fqdn,
outage.server.customer.id,
outage.server.customer.name,
)
)
except:
logging.exception("error appending message to email")
continue
msg = "\n".join(msg)
logging.info("Sending email notifying of long-running outages")
utilities.quick_send_email(
"*****@*****.**", "*****@*****.**", "%d long running outages" % outages.count(), msg
)
def apply_template_task(self, task):
"""Apply a template to one or more servers"""
params = task.parameters
server_template = masterdb.Server.get(params['template_id'])
servers = [masterdb.Server.get(s_id) for s_id in params['server_ids']]
user = masterdb.User.get(params['user_id'])
if not server_template.isTemplate(): raise Exception("Tried to apply non-template")
if user.customer != task.customer: raise Exception("Invalid user applying template")
if server_template.customer != task.customer: raise Exception("Invalid user applying template")
num_servers = 0
for server in servers:
if server.customer != task.customer: continue
server.apply_template(server_template.id, user)
num_servers += 1
# Send a confirmation email if desired
if 'email_address' in params:
to_address = params['email_address']
from_address = user.customer.brand.support_email_address
subject = "%s: %s application complete" % (user.customer.brand.name, server_template.name)
msg = """%s-
We have completed applying the template %s to %s servers. If you need any additional assistance
please contact us at %s.
""" % (user.display_name, server_template.name, num_servers, from_address)
quick_send_email(from_address, to_address, subject, msg, msg_html="", replyto="", attachments=[], bcc=[])
def send_email(log, template_name, kwargs, customer_email):
template_name = "et_" + template_name
template = globals().get(template_name, None)
if not template: raise Exception, "no template string '%s'" % template
template = Template(template)
email_msg = template.substitute(**kwargs)
if not DRY_RUN:
log.info("sending %s email to %s" % (template_name, customer_email))
# send to customer
utilities.quick_send_email(
"*****@*****.**",
[customer_email, "*****@*****.**"],
email_msg
)
else:
log.info("if this wasn't a dry run, i would've send the following email to %s:\n\n%s\n\n" % (
customer_email, email_msg
))
mp.throttle_frequency(new_freq)
if __name__ == "__main__":
parser = optparse.OptionParser(usage="USAGE: %prog [options]")
parser.add_option("-c", "--config", action="store", dest="config")
parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False)
options, args = parser.parse_args()
turbogears.update_config(configfile=options.config)
debug_email_messages = []
config = turbogears.config
tuning_param = int(config.get("throttle_constant", "18"))
if not options.debug:
notify_us_of_long_outages()
outages = get_long_running_outages()
for outage in outages:
throttle_outage(outage, tuning=tuning_param, debug=options.debug)
if options.debug and debug_email_messages:
utilities.quick_send_email(
"*****@*****.**",
"*****@*****.**",
"Dry-run outage check throttling",
"\n".join(debug_email_messages),
)
