Beispiel #1
0
 def format_clear(self, entries, **options):
     subject = options["subject"]
     replyto_address = options["replyto_address"]
     from_address = options["from_address"]
     extra_headers = options["extra_headers"]
     
     if self.customer.useNewOutageEmailTemplate():
         try:
             raw_text = masterdb.SystemEmailTemplateContent.fetch(self.customer.brand, "notifyengine2.clear").raw_text
             raw_html = masterdb.SystemEmailTemplateContent.fetch(self.customer.brand, "notifyengine2.clear").raw_html
             text_template = None
             html_template = None
         except:
             text_template = EmailText.clear_template
             html_template = self.new_template
             raw_text = raw_html = None
     else:
          text_template = EmailText.clear_template
          html_template = self.clear_template
          raw_text = raw_html = None
          
     options['raw_template'] = raw_text
     text = self.format_message(text_template, entries, **options)
     options['raw_template'] = raw_html
     html = self.format_message(html_template, entries, **options)
     msg = buildEmail(from_address, self.contact_info, subject, text, html, replyto_address, extra_headers)
     return msg.as_string()
Beispiel #2
0
 def format_broadcast(self, entries, **options):
     subject = options["subject"]
     replyto_address = options["replyto_address"]
     from_address = options["from_address"]
     extra_headers = options["extra_headers"]
     
     text = TruncatedTextPlugin.format_broadcast(self, entries, **options)
     msg = buildEmail(from_address, self.contact_info, subject, text, "", replyto_address, extra_headers)
def validate_recurring_profiles():
    profiles = PaypalProfile.selectBy(status='active', type='recurring')
    logging.info("validating %d recurring profiles" % profiles.count())
    for profile in profiles:
        amt = sum([amount for desc, amount, item in profile.customer.getFees()])
        if amt != profile.amount:
            logging.error("profile id-%d doesn't match customer's fees!" % profile.id)
            msg = "Customer %d's PayPal recurring profile id-%d amount %.2f differs from their package price with fees %.2f"
            msg = msg % (profile.id, profile.customer.id, profile.amount, amt)
            body = utilities.buildEmail('*****@*****.**', '*****@*****.**', 'PayPal error!', msg, '')
            #try: utilities.sendEmail('*****@*****.**', '*****@*****.**', body)
            #except: pass
        else:
            logging.info("profile id-%d OK" % profile.id)
    def send_exception_email(self, status, url, data, user, headers, params):
        """Send an email with the error info to the admin.

        Uses TurboMail if installed and activated, otherwise tries to send
        email with the smtplib module. The SMTP settings can be configured
        with the following configuration settings:

        error_catcher.smtp_host   - Mail server to connect to
                                    (default 'localhost')
        error_catcher.smtp_user   - User name for SMTP authentication.
                                    If unset no SMTP login is performed.
        error_catcher.smtp_passwd - Password for SMTP authentication
        """

        if not self.sender_email or not  self.admin_email:
            log.exception('Configuration error: could not send error'
              'because sender and/or admin email address is not set.')
            raise RuntimeError

        subject =  '%d ERROR on the Server [%s]' % (status, self.running_environment)
        text = ERROR_MAIL_TMPL % dict(url=url, data=data, user=user, headers=headers, params=params)

        if has_turbomail and turbogears.config.get('mail.on'):
            msg = utilities.buildEmail(self.sender_email, self.admin_email, subject, text, '')
#            msg = turbomail.Message(self.sender_email, self.admin_email, subject)
#            msg.plain = text
#            turbomail.enqueue(msg)
            utilities.sendEmail(self.sender_email, self.admin_email, msg)

        else:
            from email.MIMEMultipart import MIMEMultipart
            from email.MIMEText import MIMEText
            from email.Utils import formatdate
            msg = MIMEMultipart()
            msg['From'] = self.sender_email
            msg['To'] = self.admin_email
            msg['Date'] = formatdate(localtime=True)
            msg['Subject'] = subject
            msg.attach(MIMEText(text))
            self._send_email_smtp(self.sender_email, self.admin_email,
              msg.as_string())
Beispiel #5
0
    def format_broadcast(self, entries, **options):
        subject = options["subject"]
        replyto_address = options["replyto_address"]
        from_address = options["from_address"]
        extra_headers = options["extra_headers"]
        
        if self.customer.useNewOutageEmailTemplate():
            try:
                text_template = None
                raw_template = masterdb.SystemEmailTemplateContent.fetch(self.customer.brand, "notifyengine2.broadcast").raw_text
            except:
                text_template = self.broadcast_template
                raw_template = ""
        else:
                text_template = self.broadcast_template
                raw_template = ""

        options['raw_template'] = raw_template
        text = self.format_message(text_template, entries, **options)
        msg = buildEmail(from_address, self.contact_info, subject, text, "", replyto_address, extra_headers)
        return msg.as_string()
    def run(self):
        report_requests_output_dir = turbogears.config.get('report_requests_output_dir', '/etc/report_requests')

        while True:
            time.sleep(5.0)

            if self.shutdown_flag.isSet():
                sys.exit()
            else:
                logging.info("Looking for reports")
                report_requests = masterdb.ReportRequest.selectBy(status='new')
                if not report_requests.count(): continue
                rr = report_requests[0]
                logging.info("Generating report request %s for %s" % (rr.id, rr.request_user.display_name))
                brand_name = rr.customer.brand.name if rr.customer.brand else "Panopta"
                tempdir = tempfile.mkdtemp()

                try:
                    rr.status = 'inprocess'

                    # Build the list of servers that the report covers
                    servers = set()
                    for mp in rr.monitor_points:
                        if mp.server.status == 'active': servers.add(mp.server)
                    for sr in rr.server_resources:
                        if sr.server.status == 'active': servers.add(sr.server)
                    servers = list(servers)
                    logging.info("%s: Found %s servers for report" % (rr.id, len(servers)))

                    # Get event history
                    all_outages = []
                    for s in servers:
                        for a in masterdb.ServerResourceAnomaly.select(AND(masterdb.ServerResourceAnomaly.q.server == s,
                                                                           masterdb.ServerResourceAnomaly.q.start_time >= rr.report_start_time,
                                                                           OR(masterdb.ServerResourceAnomaly.q.end_time == None, masterdb.ServerResourceAnomaly.q.end_time <= rr.report_end_time))):
                            all_outages.append(a)
                        for o in masterdb.ServerOutage.select(AND(masterdb.ServerOutage.q.server == s,
                                                                  IN(masterdb.ServerOutage.q.status, ['confirmed', 'resolved']),
                                                                  masterdb.ServerOutage.q.start_time >= rr.report_start_time,
                                                                  OR(masterdb.ServerOutage.q.end_time == None, masterdb.ServerOutage.q.end_time <= rr.report_end_time))):
                            all_outages.append(o)
                    all_outages.sort(key=lambda o: o.start_time, reverse=True)
                    logging.info("%s: Found %s outages" % (rr.id, len(all_outages)))

                    date_formatter = rr.request_user.contact.formatLocalTime 

                    # Get servers, monitor_points, agent_resources, snmp_resources
                    servers = {}
                    for mp in rr.monitor_points:
                        try: server = mp.server
                        except: continue
                        if server not in servers: servers[server] = set([mp])
                        else: servers[server].add(mp)
                    for sr in rr.server_resources:
                        try: server = sr.server
                        except: continue
                        if sr.monitor_point:
                            if server not in servers: servers[server] = set([sr.monitor_point])
                            else: servers[server].add(sr.monitor_point)
                    servers = servers.items()
                    servers.sort(key=lambda s: s[0].fqdn.lower()) 

                    # Generating server excel files
                    book_names = []
                    overview_data = []
                    for server, monitor_points in servers:
                        logging.info("%s: Generating Excel file for %s" % (rr.id, server.formatted_name))

                        downtime = "%.2f" % (server.getTotalDownTime(rr.report_start_time, rr.report_end_time) / 60.0)
                        downtime_with_excluded = "%.2f" % (server.getTotalDownTime(rr.report_start_time, 
                                                                                   rr.report_end_time,
                                                                                   with_excluded_outages=True) / 60.0)
                        availability = "%.2f%%" % (server.getAvailability(rr.report_start_time, 
                                                                          rr.report_end_time)[0] * 100.0)
                        availability_with_excluded = "%.2f%%" % (server.getAvailability(rr.report_start_time, 
                                                                                        rr.report_end_time,
                                                                                        with_excluded_outages=True)[0] * 100.0)
                        overview_data.append([server.fqdn, downtime, availability, downtime_with_excluded, availability_with_excluded, ""])

                        # Generating detailed check results sheet
                        logging.info("%s: Generating check result sheet" % rr.id)
                        book = tablib.Databook()
                        sheet = tablib.Dataset()
                        book.add_sheet(sheet)
                        sheet.title = "Detailed Check Results"
                        sheet.append(["%s Detailed Check Results" % server.fqdn, "", "", "", ""])
                        sheet.append(["", "", "", "", ""])

                        report_duration = (rr.report_end_time - rr.report_start_time).days
                        if report_duration > 31:
                            sheet.append(["Timestamp", "Location", "Service", "Average Duration (s)", ""])
                            num_records, results = masterdb.getCheckResults_summary(monitor_points, rr.report_start_time, rr.report_end_time)
                            for monitor_point, monitor_node, check_date, result in results:
                                sheet.append([check_date, monitor_node and monitor_node.name or "", monitor_point.service_type.name, 1, result])
                        else:
                            sheet.append(["Timestamp", "Location", "Service", "Result", "Duration (s)"])
                            num_records, results = masterdb.getCheckResults_new(monitor_points, rr.report_start_time, rr.report_end_time, batch_size=1440*35)
                            for monitor_point, monitor_node, check_time, result, result_duration in results:
                                sheet.append([date_formatter(check_time), monitor_node and monitor_node.name or "", monitor_point.service_type.name, result, result_duration])
                            
                        # Generating detailed agent server resource level results sheet
                        logging.info("%s: Generating agent metrics sheet" % rr.id)
                        agent_resources = masterdb.ServerResource.select(AND(masterdb.ServerResource.q.server == server,
                                                                             masterdb.ServerResource.q.check_method == "resource.agent.panopta"))
                        if agent_resources.count():
                            sheet = tablib.Dataset()                            
                            sheet.title = "Agent Metrics"
                            sheet.append(["%s Detailed Agent Resource Results" % server.fqdn, "", ""])
                            sheet.append(["", "", ""])
                            sheet.append(["Timestamp", "Resource", "Result"]) 
                            num_records, results = masterdb.getMultipleServerResourceLevelData(agent_resources, rr.report_start_time, rr.report_end_time, batch_size=1440*35)
                            for server_resource, check_time, result in results:
                                sheet.append([date_formatter(check_time), server_resource.name, result])
                            if results: book.add_sheet(sheet)

                        # Generating detailed snmp server resource level results sheet
                        logging.info("%s: Generating SNMP metrics sheet" % rr.id)
                        snmp_resources = masterdb.ServerResource.select(AND(masterdb.ServerResource.q.server == server,
                                                                            OR(masterdb.ServerResource.q.check_method == "resource.snmp",
                                                                               masterdb.ServerResource.q.check_method == "resource.snmp.child")))
                        if snmp_resources.count():
                            sheet = tablib.Dataset()                            
                            sheet.title = "SNMP Metrics"
                            sheet.append(["%s Detailed SNMP Resource Results" % server.fqdn, "", ""])
                            sheet.append(["", "", ""])
                            sheet.append(["Timestamp", "Resource", "Result"]) 
                            num_records, results = masterdb.getMultipleServerResourceLevelData(snmp_resources, rr.report_start_time, rr.report_end_time, batch_size=1440*35)
                            for server_resource, check_time, result in results:
                                sheet.append([date_formatter(check_time), server_resource.name, result])
                            if results: book.add_sheet(sheet)

                        # Save server excel file
                        book_name = server.fqdn[:30]
                        if book_name in book_names: book_name = "%s_%s" % (server.fqdn[:30], server.id)
                        book_names.append(book_name)
                        with open(os.path.join(tempdir, '%s.xls' % book_name), 'wb') as f: f.write(book.xls)

                    # Generating the Overview excel file
                    logging.info("%s: Generating overview spreadsheet" % rr.id)
                    book = tablib.Databook()
                    sheet = tablib.Dataset()
                    book.add_sheet(sheet)
                    sheet.title = 'Overview'
                    sheet.append(["%s Monitoring Report" % brand_name, "", "", "", "", ""])
                    sheet.append(["Report covers time period %s - %s" % (date_formatter(rr.report_start_time), date_formatter(rr.report_end_time)), "", "", "", "", ""])
                    sheet.append(["", "", "", "", "", ""])
                    sheet.append(["Overview", "", "", "", "", ""])
                    sheet.append(["Server", "Downtime (minutes)", "Availability", "Downtime w/ Excluded Outages (minutes)", "Availabilty w/ Excluded outages", ""])
                    for od in overview_data: sheet.append(od)                        
                    sheet.append(["", "", "", "", "", ""])
                    sheet.append(["Event History", "", "", "", "", ""])
                    sheet.append(["Server", "Excluded?", "Start Time", "End Time", "Services", "Reason"])
                    for outage in all_outages:
                        sheet.append([outage.server.fqdn if outage.server else outage.target.name, outage.exclude_from_availability and "Yes" or "No", date_formatter(outage.start_time), date_formatter(outage.end_time) if outage.end_time else "Ongoing", str(outage), outage.getOutageReasons("; ")])
                    with open(os.path.join(tempdir, 'Overview.xls'), 'wb') as f: f.write(book.xls)

                    # Create the zip file
                    logging.info("%s: Creating zip file" % rr.id)
                    output_dir = os.path.join(report_requests_output_dir, rr.customer.brand.textkey if rr.customer.brand else 'panopta')
                    if not os.path.exists(output_dir): os.makedirs(output_dir)
                    file_name = '%s-Report-%s-%s.zip' % (brand_name.replace(" ", "_"), datetime.now().strftime("%Y%m%d"), rr.id)
                    rr.file_name = file_name
                    output_file = os.path.join(output_dir, file_name)
                    os.popen("zip -r -j %s %s" % (output_file, tempdir))  
                    shutil.rmtree(tempdir)
                    logging.info("%s: Created report %s" % (rr.id, output_file))

                    zip_size = int(os.path.getsize(output_file)/1024./1024.)

                    # Send the report email
                    try:
                        from_address = turbogears.config.get('from_address', '*****@*****.**')
                        reply_to_address = turbogears.config.get('reply_to_address', '*****@*****.**')
                        bcc_address = turbogears.config.get('bcc_address', '*****@*****.**')
                        to_address = rr.email_address
                        title = rr.name or ("Your %s monitoring report" % brand_name)
                        if zip_size > ZIP_SIZE_CUTOFF:
                            # Generated file is too big, don't send as attachment
                            template = masterdb.SystemEmailTemplateContent.fetch(rr.customer.brand, "report_request_no_attachment")
                            text = template.text.safe_substitute({'from_address': from_address, 'to_address': to_address, 'brand': brand_name, 'report_request': rr})
                            html = template.html.safe_substitute({'from_address': from_address, 'to_address': to_address, 'brand': brand_name, 'report_request': rr})
                            msg = buildEmail(from_address, to_address, title, text, html, reply_to_address)
                        else:
                            # Generated file is not too big, include as attachment
                            template = masterdb.SystemEmailTemplateContent.fetch(rr.customer.brand, "report_request")
                            text = template.text.safe_substitute({'from_address': from_address, 'to_address': to_address, 'brand': brand_name, 'report_request': rr})
                            html = template.html.safe_substitute({'from_address': from_address, 'to_address': to_address, 'brand': brand_name, 'report_request': rr})
                            msg = buildEmail(from_address, to_address, title, text, html, reply_to_address, attachments=[output_file])
                        

                        username = turbogears.config.get("smtpuser", "")
                        password = turbogears.config.get("smtppass", "")
                        s = smtplib.SMTP(turbogears.config.get('smtpserver', 'localhost'))
                        if username and password: s.login(username, password)
                        s.sendmail(from_address, [to_address, bcc_address], msg.as_string())
                        s.quit()
                        logging.info("%s: Sent report to %s" % (rr.id, to_address))
                    except:
                       logging.exception("%s: ERROR SENDING REPORT REQUEST to %s" % (rr.id, to_address))

                    rr.status = 'finalizing'
                except:
                    logging.exception("ERROR GETTING REPORT REQUEST")
                    try: rr.status = 'error'
                    except: pass

                    # Send the error email
                    try:
                        reply_to_address = from_address = "*****@*****.**"
                        to_address = "*****@*****.**"
                        title = "Report Engine Adhoc Error for report %s" % rr.id
                        html = ""
                        text = "\n".join(traceback.format_tb(sys.exc_info()[2]))
                        msg = buildEmail(from_address, to_address, title, text, html, reply_to_address)
                        username = turbogears.config.get("smtpuser", "")
                        password = turbogears.config.get("smtppass", "")
                        s = smtplib.SMTP(turbogears.config.get('smtpserver', 'localhost'))
                        if username and password: s.login(username, password)
                        s.sendmail(from_address, [to_address], msg.as_string())
                        s.quit()
                    except:
                        pass
    def buildMessage(self, notice_type, channel, outages, anomalies, replyto_address, from_address):
        """
        Construct an alert message for the given list of outages and anomalies, to be sent to the given channel.

        General process is to construct fragments for each item to be reported on, then assemble them all
        together into the final message.  Need to take into consideration whether we're building text email,
        HTML email, short text email or SMS message, which is determined by the channel.
        """

        customer = channel.contact.customer
        brand = customer.brand
        brand_name = "Panopta"
        if brand and brand.isWhiteLabel():
            brand_name = brand.name

        shorttext_body = u""
        text_body = u""
        html_body = u""

        # Make a list of server names to notify, for use in the subject header
        items = outages + anomalies
        server_names = set()
        for o in outages:
            try:
                target = o.outage_target
            except:
                continue
            server_names.add(target.name)
        for a in anomalies:
            server_names.add(a.server.name)
        server_names = list(server_names)
        server_names.sort()

        # Construct the message body
        if channel.contact_type.textkey in ('sms', 'email.shorttext'):
            
            # Figure out the limit on the message length - shorter for SMS to avoid truncation
            length_limit = 148
            if channel.contact_type.textkey == 'email.shorttext': length_limit = 350

            # Build up a list of all of the things to notify about
            fragments = []
            for o in outages:
                notice_time = o.start_time
                if notice_type == ALL_CLEAR:
                    notice_time = o.end_time
                fragments.append(_("%(server_name)s %(notice_type)s at %(start_time)s: %(service_list)s") % \
                                     {'server_name': o.outage_target.name, 
                                      'notice_type': _(notice_type), 
                                      'start_time': channel.contact.formatLocalTime(notice_time, hide_same_day=1),
                                      'service_list': o.getServiceDisplayNames(join_str=',')})
            for a in anomalies:
                notice_time = a.start_time
                notice_desc = a.server_resource_threshold.getAnomalyDescription(short=1)
                if notice_type == ALL_CLEAR:
                    notice_time = a.end_time
                    notice_desc = a.server_resource_threshold.getAllClearDescription(short=1)
                fragments.append(_("%(server_name)s %(notice_type)s at %(start_time)s") % {'server_name': a.server.name, 
                                                                                           'notice_type': notice_desc,
                                                                                           'start_time': channel.contact.formatLocalTime(notice_time, hide_same_day=1)})

            # Complete the message by including as many fragments as possible while staying under 
            # the length limit.
            shorttext_body = fragments.pop(0)
            while fragments and (len(shorttext_body) + len(fragments[0]) + 2 < length_limit):
                shorttext_body += "; %s" % fragments.pop(0)
            if fragments:
                shorttext_body += _(" + %(count)s more") % {'count': len(fragments)}

        else:
            
            # Build a list of formatted things to report
            fragments = []
            for o in outages:
                fragments.append(self.buildOutageFragment(notice_type, o, customer, channel))
            for a in anomalies:
                fragments.append(self.buildAnomalyFragment(notice_type, a, customer, channel))
                                
            # Get the template fragments, in case the customer has customized them
            header = customer.getTemplateFragment('email.header')
            if notice_type == OUTAGE:
                intro = customer.getTemplateFragment('email.outage.notice.intro')
                conclusion = customer.getTemplateFragment('email.outage.notice.conclusion')
            elif notice_type == ALL_CLEAR:
                intro = customer.getTemplateFragment('email.outage.allclear.intro')
                conclusion = customer.getTemplateFragment('email.outage.allclear.conclusion')
            copyright = customer.getTemplateFragment('email.copyright')

            # Build the text rendering of the list, as we can't easily do this with interpolation
            alert_list = ""
            for (head, sub_list) in fragments:
                alert_list += " o %s\n" % head
                for line in sub_list:
                    alert_list += "   - %s\n" % line
                alert_list += "\n"

            # Build the text body
            if header: text_body += header.body_text + "\n\n"
            if intro: text_body += intro.body_text + "\n\n"

            template = _(unicode(outage_text_template))
            if notice_type == ALL_CLEAR: template = _(unicode(allclear_text_template))
            text_body += template % {'alerts': alert_list,
                                     'brand_name': brand_name}
            if conclusion: text_body += conclusion.body_text + "\n\n"
            if copyright: text_body += copyright.body_text

            if channel.contact_type.textkey == 'email.html':
                
                # Build the HTML body
                template_data = {'alerts': fragments,
                                 'brand_name': brand_name,
                                 'source': unicode(_(outage_html_template)) }
                if notice_type == ALL_CLEAR: template_data['source'] = unicode(_(allclear_html_template))

                if header: html_body += unicode(header.body_html) + u"\n\n"
                if intro: html_body += unicode(intro.body_html) + u"\n\n"

#                tmpl = MarkupTemplate(unicode(_(outage_html_template)))
#                stream = tmpl.generate(alerts=fragments, brand_name=brand_name)
#                frag = stream.render('xhtml')
#                frag = apply(kid.Template, [], template_data).serialize(encoding='utf-8')
                from kid import Template
                t=Template(unicode(_(outage_html_template)),
                           brand_name = brand_name,
                           alerts = fragments)
                frag = unicode(t.serialize(output='xhtml', encoding='utf-8'), 'utf8')

                html_body += frag
                if conclusion: html_body += conclusion.body_html + "\n\n"
                if copyright: html_body += copyright.body_html

        # Build the list of extra headers to embed in the message
        extra_headers = []
        if brand and brand.isWhiteLabel():
            extra_headers.append(('X-Panopta-PartnerKey', customer.partner_key))
            extra_headers.append(('X-Panopta-Package', customer.package.textkey))
        extra_headers.append(('X-Panopta-OutageStatus', notice_type==ALL_CLEAR and 'all-clear' or 'outage'))
        extra_headers.append(('X-Panopta-OutageId', replyto_address.split('@')[0]))  # Grab the email hash from the replyto address
        for server_name in server_names:
	    logging.critical("Adding server name %s" % server_name)
            extra_headers.append(('X-Panopta-ServerFQDN', server_name))
            
        # Build the subject header
        subject = "%s %s: %s" % (brand_name, _(notice_type), server_names.pop(0))
        while server_names and len(subject) < 80:
            subject += "; %s" % server_names.pop(0)
        if server_names:
            subject += _(" + %(count)s more") % {'count': len(server_names)}

        # Build the message object
        if channel.contact_type.textkey == 'sms':
            # If we're sending an SMS, just return the body - there's nothing else to be done
            return shorttext_body
        elif channel.contact_type.textkey == 'email.shorttext':
            msg = buildEmail(from_address, channel.email_address, subject, shorttext_body, "", replyto_address, extra_headers)
        elif channel.contact_type.textkey == 'email.text':
            msg = buildEmail(from_address, channel.email_address, subject, text_body, "", replyto_address, extra_headers)
        else:
            msg = buildEmail(from_address, channel.email_address, subject, text_body, html_body, replyto_address, extra_headers)

        return msg.as_string()
def sendReport(contact_channel, start, end, title, frequency, server_groups, include_all_servers, show_average_response_time):

    contact = contact_channel.contact
    customer = contact.customer
    
    overall_avail = 0.
    total_outages = 0
    total_downtime = timedelta(seconds=0)
    num_servers = 0
    servers = []

    server_groups.sort(key=operator.attrgetter('nested_name'))

    for group in server_groups:

        group_servers = []
        for server in group.compound_services + group.servers:
            if server.status != 'active': continue
            (avail, outages) = server.getAvailability(start, end)
            overall_avail += avail
            outage_details = {}

            # Extract the availability information from the list of outages
            for (mp, num_outages, service_avail, downtime) in outages:
                total_outages += num_outages
                total_downtime += downtime
                avail_string = availabilityFormatString(service_avail*100., customer) % (100.*service_avail)

                if mp: 
                    key = mp.id
                    mp_name = mp.service_type.name
                    if mp.name: mp_name += " : %s" % mp.name
                else: 
                    key = "cs%s" % server.id
                    mp_name = ""

                if mp and show_average_response_time:
                    avg_response_time = customer.getAvailabilityFormat() % mp.getAverageResponseTime(start, end)
                else:
                    avg_response_time = ''
                outage_details[key] = (mp_name, num_outages, avail_string, downtime, avg_response_time)

            # Get average response time information for any servers that didn't have outages
            if server.__class__.__name__ != 'CompoundService':
                for mp in server.active_monitor_points:
                    if not outage_details.has_key(mp.id):
                        if show_average_response_time:
                            avg_response_time = customer.getAvailabilityFormat() % mp.getAverageResponseTime(start, end)
                        else:
                            avg_response_time = 0.
                        mp_name = mp.service_type.name
                        if mp.name: mp_name += unicode(" - %s" % mp.name)
                        outage_details[mp.id] = (mp_name,
                                                 0,
                                                 availabilityFormatString(100., customer) % 100., 
                                                 timedelta(hours=0), 
                                                 avg_response_time)
                    
            if include_all_servers or (avail != 1.0 or outages):
                # Only show a specific server if there's been an outage, unless the customer has 
                # chosen to show all servers
                avail_string = availabilityFormatString(avail*100., customer) % (100.*avail)
                group_servers.append((server, avail_string, sorted(outage_details.values())))
    
            num_servers += 1
        if group_servers: servers.append((group, group_servers))

    if num_servers:
        overall_avail = overall_avail*100./(1.0*num_servers)
        format_str = availabilityFormatString(overall_avail, customer)
        overall_avail = format_str % overall_avail
    else:
        # Don't send the email if they have no servers to report on
        return

    if total_outages:
        average_downtime = formatTimedelta(total_downtime/total_outages)
    else:
        average_downtime = unicode(_("N/A"))
    total_downtime = formatTimedelta(total_downtime)
    
    report_type = string.capwords(frequency)
   
    from_address = turbogears.config.get('from_address', '*****@*****.**')
    reply_to_address = turbogears.config.get('reply_to_address', '*****@*****.**')
    bcc_address = turbogears.config.get('bcc_address', '*****@*****.**')
    to_address = contact_channel.email_address

    if customer.brand and customer.brand.isWhiteLabel() and customer.brand.support_email_address:
        from_address = customer.brand.support_email_address
        reply_to_address = customer.brand.support_email_address

    # Get the fragments to use for building the report content
    html_start = customer.getTemplateFragment('email.report.header')
    header = customer.getTemplateFragment('email.header')
    intro = customer.getTemplateFragment('email.report.intro')
    conclusion = customer.getTemplateFragment('email.report.conclusion')
    copyright = customer.getTemplateFragment('email.copyright')

    # Generate the HTML version of the report - Genshi for the actual report body,
    # glued together with other fragments
    tmpl = MarkupTemplate(open("ReportTemplate.html"))
    tmpl.filters.insert(0, Translator(_))
    stream = tmpl.generate(overall_avail=overall_avail,
                           controlpanel_url=customer.getControlPanelURL().rstrip('/'),
                           servers=servers, 
                           title=title,
                           report_type=report_type,
                           total_downtime=total_downtime,
                           average_downtime=average_downtime, 
                           total_outages=total_outages, 
                           num_servers=num_servers,
                           formatTimedelta=formatTimedelta,
                           show_average_response_time=show_average_response_time)
    
    report_html = stream.render('html', None)

    if html_start:
        html = html_start.body_html
    else:
        html = HTML_HEADER

    if header: html += header.body_html 
    html += "<h1>%s</h1>" % title
    if intro: html += intro.body_html
    html += report_html
    if conclusion: html += conclusion.body_html
    if copyright: html += copyright.body_html

    # Generate the text version of the report
    text_tmpl = TextTemplate(unicode(open("ReportTemplate.txt").read(), 'utf8'))
    text_tmpl.filters.insert(0, Translator(_))
    text_stream = text_tmpl.generate(overall_avail=overall_avail,
                                     servers=servers,
                                     title=title,
                                     report_type=report_type,
                                     total_downtime=total_downtime,
                                     average_downtime=average_downtime, 
                                     total_outages=total_outages,
                                     num_servers=len(customer.servers),
                                     formatTimedelta=formatTimedelta,
                                     show_average_response_time=show_average_response_time,
                                     center=center)
    report_text = text_stream.render('text')
    text = u""
    if header: text += header.body_text + "\n"
    text += "\n%s\n===============================================================================\n\n" % title
    if intro: text += intro.body_text + "\n\n"

    text += report_text + "\n\n"
    if conclusion: text += conclusion.body_text + "\n\n"
    if copyright: text += copyright.body_text + "\n"

    msg = buildEmail(from_address, to_address, title, text, html, reply_to_address)

#     msg = MIMEMultipart('related')
#     msg['Subject'] = title
#     msg['From'] = from_address
#     msg['To'] = to_address
#     msg['Reply-To'] = reply_to_address
#     msg.preamble = "This is a multi-part message in MIME format."
   
#     # Encapsulate the plain and HTML versions of the message body in an
#     # 'alternative' part, so message agents can decide which they want to display.
#     msgAlternative = MIMEMultipart('alternative')
#     msg.attach(msgAlternative)
                
#     msg_text = MIMEText(text)
#     msgAlternative.attach(msg_text)
    
#     msgAlternative.attach(MIMEText(html, 'html'))
    
    username = turbogears.config.get("smtpuser", "")
    password = turbogears.config.get("smtppass", "")
    
    s = smtplib.SMTP(turbogears.config.get('smtpserver', 'localhost'))
    if username and password:
        s.login(username, password)
    s.sendmail(from_address, [to_address, bcc_address], msg.as_string())
    s.quit()
    logging.info("Sent report to %s" % to_address)
                title = unicode(_("%(brand_name)s Monthly Availability Report for %(month)s") % {'brand_name': brand_name, 
                                                                                                 'month': prev_day.strftime("%B, %Y")})
                new_end_time = end + timedelta(days=days_in_next_month)
            else:
                continue
    
            logging.info("Generating %s report for %s" % (frequency, contact_channel.email_address))
            if customer.status == 'active':
                sendReport(contact_channel, start, end, title, frequency, report.getServerGroups(), 
                           report.include_all_servers, report.show_average_response_time)
            report.next_end_time = new_end_time
            
        except:
            if contact_channel: logging.exception("ERROR SENDING REPORT to %s" % contact_channel.email_address)
            else: logging.exception("ERROR GETTING REPORT")

            try:
                reply_to_address = from_address = "*****@*****.**"
                to_address = "*****@*****.**"
                title = "Report Engine Error"
                html = text = traceback.format_tb(sys.exc_info()[2])
                msg = buildEmail(from_address, to_address, title, text, html, reply_to_address)
                username = turbogears.config.get("smtpuser", "")
                password = turbogears.config.get("smtppass", "")
                s = smtplib.SMTP(turbogears.config.get('smtpserver', 'localhost'))
                if username and password: s.login(username, password)
                s.sendmail(from_address, [to_address], msg.as_string())
                s.quit()
            except:
                pass