def main():
    """Main entry."""
    # Clear all loggers to make sure the following basicConfig take effect.
    logging.shutdown()
    reload(logging)
    logging.basicConfig(format='%(asctime)s %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S', level=logging.DEBUG)

    with ts_mon_config.SetupTsMonGlobalState(service_name='kill_slow_queries',
                                             indirect=True):
        count = 0
        parser, options, args = parse_options()
        if not verify_options_and_args(options, args):
            parser.print_help()
            return 1
        try:
            while True:
                result_log_strs, count = kill_slow_queries(
                    options.user, options.password, options.timeout)
                if result_log_strs:
                    gmail_lib.send_email(
                        options.mail,
                        'Successfully killed slow autotest db queries',
                        'Below are killed queries:\n%s' % result_log_strs)
                    m = 'chromeos/autotest/afe_db/killed_slow_queries'
                    metrics.Counter(m).increment_by(count)
                time.sleep(options.timeout)
        except Exception as e:
            m = 'chromeos/autotest/afe_db/failed_to_kill_query'
            metrics.Counter(m).increment()
            logging.error('Failed to kill slow db queries.\n%s', e)
            raise
Exemplo n.º 2
0
def send_email(bug, bug_template):
    """Send email to the owner and cc's to notify the TestBug.

    @param bug: TestBug instance.
    @param bug_template: A template dictionary specifying the default bug
                         filing options for failures in this suite.
    """
    autotest_stats.Counter(EMAIL_COUNT_KEY % 'total').increment()
    to_set = set(bug.cc) if bug.cc else set()
    if bug.owner:
        to_set.add(bug.owner)
    if bug_template.get('cc'):
        to_set = to_set.union(bug_template.get('cc'))
    if bug_template.get('owner'):
        to_set.add(bug_template.get('owner'))
    recipients = ', '.join(to_set)
    try:
        gmail_lib.send_email(
            recipients,
            bug.title(),
            bug.summary(),
            retry=False,
            creds_path=site_utils.get_creds_abspath(EMAIL_CREDS_FILE))
    except Exception:
        autotest_stats.Counter(EMAIL_COUNT_KEY % 'fail').increment()
        raise
Exemplo n.º 3
0
def send_email(bug, bug_template):
    """Send email to the owner and cc's to notify the TestBug.

    @param bug: TestBug instance.
    @param bug_template: A template dictionary specifying the default bug
                         filing options for failures in this suite.
    """
    to_set = set(bug.cc) if bug.cc else set()
    if bug.owner:
        to_set.add(bug.owner)
    if bug_template.get('cc'):
        to_set = to_set.union(bug_template.get('cc'))
    if bug_template.get('owner'):
        to_set.add(bug_template.get('owner'))
    recipients = ', '.join(to_set)
    if not recipients:
        logging.warning('No owner/cc found. Will skip sending a mail.')
        return
    success = False
    try:
        gmail_lib.send_email(
            recipients,
            bug.title(),
            bug.summary(),
            retry=False,
            creds_path=site_utils.get_creds_abspath(EMAIL_CREDS_FILE))
        success = True
    finally:
        (metrics.Counter('chromeos/autotest/errors/send_bug_email').increment(
            fields={'success': success}))
Exemplo n.º 4
0
def main():
    """Main entry."""
    logging.basicConfig(format='%(asctime)s %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S',
                        level=logging.DEBUG)

    with ts_mon_config.SetupTsMonGlobalState(service_name='kill_slow_queries',
                                             indirect=True):
        count = 0
        parser, options, args = parse_options()
        if not verify_options_and_args(options, args):
            parser.print_help()
            return 1
        try:
            while True:
                result_log_strs, count = kill_slow_queries(
                    options.user, options.password, options.timeout)
                if result_log_strs:
                    gmail_lib.send_email(
                        options.mail,
                        'Successfully killed slow autotest db queries',
                        'Below are killed queries:\n%s' % result_log_strs)
                    m = 'chromeos/autotest/afe_db/killed_slow_queries'
                    metrics.Counter(m).increment_by(count)
                time.sleep(options.timeout)
        except Exception as e:
            logging.error('Failed to kill slow db queries.\n%s', e)
            gmail_lib.send_email(
                options.mail, 'Failed to kill slow autotest db queries.',
                ('Error occurred during killing slow db queries:\n%s\n'
                 'Detailed logs can be found in /var/log/slow_queries.log on db'
                 ' backup server.\nTo avoid db crash, please check ASAP.') % e)
            raise
Exemplo n.º 5
0
def main():
    """Main entry."""
    logging.basicConfig(format='%(asctime)s %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S',
                        level=logging.DEBUG)

    parser, options, args = parse_options()
    if not verify_options_and_args(options, args):
        parser.print_help()
        return 1
    try:
        result_log_strs = kill_slow_queries(options.user, options.password,
                                            options.timeout)
        if result_log_strs:
            gmail_lib.send_email(
                options.mail, 'Successfully killed slow autotest db queries',
                'Below are killed queries:\n%s' % result_log_strs)
    except Exception as e:
        logging.error('Failed to kill slow db queries.\n%s', e)
        gmail_lib.send_email(
            options.mail, 'Failed to kill slow autotest db queries.',
            ('Error occurred during killing slow db queries:\n%s\n'
             'Detailed logs can be found in /var/log/slow_queries.log on db '
             'backup server.\nTo avoid db crash, please check ASAP.') % e)
        raise
Exemplo n.º 6
0
def send_notification_email(email_list, title, msg):
    """Send notification to all email addresses in email list.

    @param email_list: a email address list which receives notification email,
        whose format is like:
            [[email protected], [email protected], [email protected],...]
        so that users could also specify multiple email addresses by using
        config '--email' or '-e'.
    @param title: the title of the email to be sent.
    @param msg: the content of the email to be sent.
    """
    gmail_lib.send_email(','.join(email_list), title, msg)
def main():
    """main script. """
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--span',
        type=int,
        dest='span',
        default=1,
        help=('Number of hours that stats should be collected. '
              'If it is set to 24, the end time of stats being '
              'collected will set to the mid of the night. '
              'Default is set to 1 hour.'))
    parser.add_argument('-e',
                        '--email',
                        dest='email',
                        default=None,
                        help='Email any errors to the given email address.')
    options = parser.parse_args()

    boards = host_label_utils.get_all_boards()
    pools = ['bvt', 'suites', 'cq']

    if options.span == 24:
        today = datetime.combine(date.today(), datetime.min.time())
        end_time = time_utils.to_epoch_time(today)
    else:
        now = datetime.now()
        end_time = datetime(year=now.year,
                            month=now.month,
                            day=now.day,
                            hour=now.hour)
        end_time = time_utils.to_epoch_time(end_time)

    start_time = end_time - timedelta(hours=options.span).total_seconds()
    print('Collecting host stats from %s to %s...' %
          (time_utils.epoch_time_to_date_string(start_time),
           time_utils.epoch_time_to_date_string(end_time)))

    errors = []
    if not boards:
        errors.append('Error! No board found in metadb.')
    for board in boards:
        for pool in pools:
            error = report_stats(board, pool, start_time, end_time,
                                 options.span)
            if error:
                errors.append(error)
    if options.email and errors:
        gmail_lib.send_email(options.email,
                             'Error occured when collecting host stats.',
                             '\n'.join(errors))
Exemplo n.º 8
0
    def send_email(self, to_string, subject, body):
        """Mails out emails to the addresses listed in to_string.

        @param to_string: is split into a list which can be delimited by any of:
                          ';', ',', ':' or any whitespace.
        @param subject: String, email subject.
        @param body: String, message body
        """
        # Create list from string removing empty strings from the list.
        to_list = [x for x in re.split('\s|,|;|:', to_string) if x]
        if not to_list:
            return
        to_string = ','.join(to_list)
        try:
            gmail_lib.send_email(to_string, subject, body)
        except Exception:
            logging.exception('Sending email failed:')
def _send_email(arguments, tag, subject, recipients, body):
    """Send an inventory e-mail message.

    The message is logged in the selected log directory using `tag`
    for the file name.

    If the --print option was requested, the message is neither
    logged nor sent, but merely printed on stdout.

    @param arguments   Parsed command-line options.
    @param tag         Tag identifying the inventory for logging
                       purposes.
    @param subject     E-mail Subject: header line.
    @param recipients  E-mail addresses for the To: header line.
    @param body        E-mail message body.

    """
    logging.debug('Generating email: "%s"', subject)
    all_recipients = ', '.join(recipients)
    report_body = '\n'.join([
            'To: %s' % all_recipients,
            'Subject: %s' % subject,
            '', body, ''])
    if arguments.debug:
        print report_body
    else:
        filename = os.path.join(arguments.logdir, tag)
        try:
            report_file = open(filename, 'w')
            report_file.write(report_body)
            report_file.close()
        except EnvironmentError as e:
            logging.error('Failed to write %s:  %s', filename, e)
        try:
            gmail_lib.send_email(all_recipients, subject, body)
        except Exception as e:
            logging.error('Failed to send e-mail to %s:  %s',
                          all_recipients, e)
Exemplo n.º 10
0
def main():
    """Entry point for test_push script."""
    arguments = parse_arguments()
    updated_repo_heads = get_head_of_repos(UPDATED_REPOS)
    updated_repo_msg = '\n'.join(
        ['%s: %s' % (k, v) for k, v in updated_repo_heads.iteritems()])

    try:
        # Use daemon flag will kill child processes when parent process fails.
        use_daemon = not arguments.continue_on_failure
        # Verify all the DUTs at the beginning of testing push.
        reverify_all_push_duts()
        time.sleep(15)  # Wait 15 secs for the verify test to start.
        check_dut_inventory(arguments.num_duts, arguments.pool)
        queue = multiprocessing.Queue()

        push_to_prod_suite = multiprocessing.Process(
            target=test_suite_wrapper,
            args=(queue, PUSH_TO_PROD_SUITE, EXPECTED_TEST_RESULTS, arguments))
        push_to_prod_suite.daemon = use_daemon
        push_to_prod_suite.start()

        # TODO(dshi): Remove following line after crbug.com/267644 is fixed.
        # Also, merge EXPECTED_TEST_RESULTS_AU to EXPECTED_TEST_RESULTS
        # AU suite will be on shard until crbug.com/634049 is fixed.
        au_suite = multiprocessing.Process(target=test_suite_wrapper,
                                           args=(queue, AU_SUITE,
                                                 EXPECTED_TEST_RESULTS_AU,
                                                 arguments, True))
        au_suite.daemon = use_daemon
        au_suite.start()

        # suite test with --create_and_return flag
        asynchronous_suite = multiprocessing.Process(
            target=test_suite_wrapper,
            args=(queue, DUMMY_SUITE, EXPECTED_TEST_RESULTS_DUMMY, arguments,
                  False, True))
        asynchronous_suite.daemon = True
        asynchronous_suite.start()

        # Test suite for testbed
        testbed_suite = multiprocessing.Process(
            target=test_suite_wrapper,
            args=(queue, TESTBED_SUITE, EXPECTED_TEST_RESULTS_TESTBED,
                  arguments, False, False, True))
        testbed_suite.daemon = use_daemon
        testbed_suite.start()

        while (push_to_prod_suite.is_alive() or au_suite.is_alive()
               or asynchronous_suite.is_alive() or testbed_suite.is_alive()):
            check_queue(queue)
            time.sleep(5)

        check_queue(queue)

        push_to_prod_suite.join()
        au_suite.join()
        asynchronous_suite.join()
        testbed_suite.join()

        # All tests pass, push prod-next branch for UPDATED_REPOS.
        push_prod_next_branch(updated_repo_heads)
    except Exception as e:
        print 'Test for pushing to prod failed:\n'
        print str(e)
        # Abort running jobs when choose not to continue when there is failure.
        if not arguments.continue_on_failure:
            for suite_id in all_suite_ids:
                if AFE.get_jobs(id=suite_id, finished=False):
                    AFE.run('abort_host_queue_entries', job=suite_id)
        # Send out email about the test failure.
        if arguments.email:
            gmail_lib.send_email(
                arguments.email,
                'Test for pushing to prod failed. Do NOT push!',
                ('Test CLs of the following repos failed. Below are the '
                 'repos and the corresponding test HEAD.\n\n%s\n\n.'
                 'Error occurred during test:\n\n%s\n\n'
                 'All logs have been saved to '
                 '/var/log/test_push/test_push.log on push master. Detail '
                 'debugging info can be found at go/push-to-prod' %
                 (updated_repo_msg, str(e)) + '\n'.join(run_suite_output)))
        raise
    finally:
        # Reverify all the hosts
        reverify_all_push_duts()

    message = (
        '\nAll tests are completed successfully, the prod branch of the '
        'following repos ready to be pushed to the hash list below.\n'
        '%s\n\n\nInstructions for pushing to prod are available at '
        'https://goto.google.com/autotest-to-prod' % updated_repo_msg)
    print message
    # Send out email about test completed successfully.
    if arguments.email:
        gmail_lib.send_email(
            arguments.email, 'Test for pushing to prod completed successfully',
            message)
Exemplo n.º 11
0
def main():
    """Entry point for test_push script."""
    arguments = parse_arguments()

    try:
        # Close existing bugs. New bug should be filed in dummy_Fail test.
        old_issue_ids = close_bug()

        queue = multiprocessing.Queue()

        push_to_prod_suite = multiprocessing.Process(
                target=test_suite_wrapper,
                args=(queue, PUSH_TO_PROD_SUITE, EXPECTED_TEST_RESULTS,
                      arguments))
        push_to_prod_suite.start()

        # TODO(dshi): Remove following line after crbug.com/267644 is fixed.
        # Also, merge EXPECTED_TEST_RESULTS_AU to EXPECTED_TEST_RESULTS
        au_suite = multiprocessing.Process(
                target=test_suite_wrapper,
                args=(queue, AU_SUITE, EXPECTED_TEST_RESULTS_AU,
                      arguments))
        au_suite.start()

        shard_suite = multiprocessing.Process(
                target=test_suite_wrapper,
                args=(queue, DUMMY_SUITE, EXPECTED_TEST_RESULTS_DUMMY,
                      arguments, True))
        shard_suite.start()

        # suite test with --create_and_return flag
        asynchronous_suite = multiprocessing.Process(
                target=test_suite_wrapper,
                args=(queue, DUMMY_SUITE, EXPECTED_TEST_RESULTS_DUMMY,
                      arguments, True, True))
        asynchronous_suite.start()

        bug_filing_checked = False
        while (push_to_prod_suite.is_alive() or au_suite.is_alive() or
               shard_suite.is_alive() or asynchronous_suite.is_alive()):
            check_queue(queue)
            # Check bug filing results to fail early if bug filing failed.
            if not bug_filing_checked and not push_to_prod_suite.is_alive():
                check_bug_filed_and_deduped(old_issue_ids)
                bug_filing_checked = True
            time.sleep(5)

        check_queue(queue)

        push_to_prod_suite.join()
        au_suite.join()
        shard_suite.join()
        asynchronous_suite.join()
    except Exception as e:
        print 'Test for pushing to prod failed:\n'
        print str(e)
        # Send out email about the test failure.
        if arguments.email:
            gmail_lib.send_email(
                    arguments.email,
                    'Test for pushing to prod failed. Do NOT push!',
                    ('Errors occurred during the test:\n\n%s\n\n' % str(e) +
                     'run_suite output:\n\n%s' % '\n'.join(run_suite_output)))
        raise

    message = ('\nAll tests are completed successfully, prod branch is ready to'
               ' be pushed.')
    print message
    # Send out email about test completed successfully.
    if arguments.email:
        gmail_lib.send_email(
                arguments.email,
                'Test for pushing to prod completed successfully',
                message)