Exemplo n.º 1
0
def inner_loop(args):
    if not args.data_url:
        logging.warn('No /data url passed, will write to builder_alerts.json')

    if args.use_cache:
        requests_cache.install_cache('failure_stats')
    else:
        requests_cache.install_cache(backend='memory')

    # FIXME: gatekeeper_config should find gatekeeper.json for us.
    gatekeeper_path = os.path.abspath(args.gatekeeper)
    logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
    gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)

    gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
    logging.debug('Processing gatekeeper trees json: %s',
                  gatekeeper_trees_path)
    gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
        gatekeeper_trees_path)

    master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
    start_time = datetime.datetime.utcnow()

    cache = buildbot.DiskCache(CACHE_PATH)

    old_alerts = {}
    if args.data_url:
        try:
            old_alerts_raw = requests.get(args.data_url[0]).json()
        except ValueError:
            logging.debug('No old alerts found.')
        else:
            # internal-alerts will have a redirect instead of alerts if you're
            # signed in.
            if 'alerts' in old_alerts_raw:
                for alert in old_alerts_raw['alerts']:
                    master = alert['master_url']
                    builder = alert['builder_name']
                    step = alert['step_name']
                    reason = alert['reason']
                    alert_key = alert_builder.generate_alert_key(
                        master, builder, step, reason)

                    if alert_key in old_alerts:
                        logging.critical(
                            'Incorrectly overwriting an alert reason from the'
                            ' old alert data. master: %s, builder: %s, step: %s, reason:'
                            ' %s' % (master, builder, step, reason))

                    old_alerts[alert_key] = alert

    latest_builder_info = {}
    stale_builder_alerts = []
    missing_masters = []
    alerts = []
    suspected_cls = []

    pool = multiprocessing.Pool(processes=args.processes)
    master_datas = pool.map(
        SubProcess(cache, old_alerts, args.builder_filter, args.jobs),
        master_urls)
    pool.close()
    pool.join()

    for data in master_datas:
        # TODO(ojan): We should put an alert in the JSON for this master so
        # we can show that the master is down in the sheriff-o-matic UI.
        if not data[0]:
            missing_masters.extend([data[3]])
            continue
        alerts.extend(data[0])
        latest_builder_info.update(data[1])
        stale_builder_alerts.extend(data[2])

    logging.info('Fetch took: %s seconds.',
                 (datetime.datetime.utcnow() - start_time).total_seconds())

    alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
                                                      gatekeeper_trees)
    stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
        stale_builder_alerts, gatekeeper, gatekeeper_trees)

    alerts = analysis.assign_keys(alerts)
    reason_groups = analysis.group_by_reason(alerts)
    range_groups = analysis.merge_by_range(reason_groups)

    if args.findit_api_url and alerts:
        suspected_cls = query_findit(args.findit_api_url, alerts)

    data = {
        'alerts': alerts,
        'suspected_cls': suspected_cls,
        'reason_groups': reason_groups,
        'range_groups': range_groups,
        'latest_builder_info': latest_builder_info,
        'stale_builder_alerts': stale_builder_alerts,
        'missing_masters': missing_masters,
    }

    if not args.data_url:
        with open('builder_alerts.json', 'w') as f:
            f.write(json.dumps(data, indent=1))

    ret = True

    json_data = json.dumps(data)
    logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
    s = cStringIO.StringIO()
    with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
        g.write(json_data)
    gzipped_data = s.getvalue()

    for url in args.data_url:
        logging.info('POST %s alerts (%s bytes compressed) to %s', len(alerts),
                     len(gzipped_data), url)
        resp = requests.post(url,
                             data=gzipped_data,
                             headers={'content-encoding': 'gzip'})
        try:
            resp.raise_for_status()
        except requests.HTTPError as e:
            logging.error('POST to %s failed! %d %s, %s, %s', url,
                          resp.status_code, resp.reason, resp.content, e)
            ret = False

    return ret
Exemplo n.º 2
0
def inner_loop(args):
  old_api_endpoint = string_helpers.slash_join(args.api_endpoint_prefix,
      args.old_api_path) if args.old_api_path else None
  if not old_api_endpoint:
    logging.warn(
        'No /data url passed, will write to builder_alerts.json. JSON posted '
        'to new API endpoints will be written to builder_alerts_<tree>.json '
        'files.')

  if args.use_cache:
    requests_cache.install_cache('failure_stats')
  else:
    requests_cache.install_cache(backend='memory')

  # FIXME: gatekeeper_config should find gatekeeper.json for us.
  gatekeeper_path = os.path.abspath(args.gatekeeper)
  logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
  gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)

  gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
  logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)
  gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
      gatekeeper_trees_path)

  master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
  start_time = datetime.datetime.utcnow()

  cache = buildbot.DiskCache(CACHE_PATH)

  old_alerts = {}
  if old_api_endpoint:
    try:
      old_alerts_raw = requests.get(old_api_endpoint).json()
    except ValueError:
      logging.debug('No old alerts found.')
    else:
      # internal-alerts will have a redirect instead of alerts if you're
      # signed in.
      if 'alerts' in old_alerts_raw:
        for alert in old_alerts_raw['alerts']:
          master = alert['master_url']
          builder = alert['builder_name']
          step = alert['step_name']
          reason = alert['reason']
          alert_key = alert_builder.generate_alert_key(
              master, builder, step, reason)

          if alert_key in old_alerts:
            logging.critical(
                'Incorrectly overwriting an alert reason from the'
                ' old alert data. master: %s, builder: %s, step: %s, reason:'
                ' %s' % (master, builder, step, reason))

          old_alerts[alert_key] = alert

  latest_builder_info = {}
  stale_builder_alerts = []
  missing_masters = []
  alerts = []
  suspected_cls = []

  pool = multiprocessing.Pool(processes=args.processes)
  master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
                                     args.jobs), master_urls)
  pool.close()
  pool.join()

  for data in master_datas:
    # TODO(ojan): We should put an alert in the JSON for this master so
    # we can show that the master is down in the sheriff-o-matic UI.
    if not data[0]:
      missing_masters.extend([data[3]])
      continue
    alerts.extend(data[0])
    latest_builder_info.update(data[1])
    stale_builder_alerts.extend(data[2])

  logging.info('Fetch took: %s seconds.',
               (datetime.datetime.utcnow() - start_time).total_seconds())

  alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
                                                    gatekeeper_trees)
  stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
      stale_builder_alerts, gatekeeper, gatekeeper_trees)

  alerts = analysis.assign_keys(alerts)
  reason_groups = analysis.group_by_reason(alerts)
  range_groups = analysis.merge_by_range(reason_groups)

  if args.findit_api_url and alerts:
    suspected_cls = query_findit(args.findit_api_url, alerts)

  data = {
      'alerts': alerts,
      'suspected_cls': suspected_cls,
      'reason_groups': reason_groups,
      'range_groups': range_groups,
      'latest_builder_info': latest_builder_info,
      'stale_builder_alerts': stale_builder_alerts,
      'missing_masters': missing_masters,
  }

  if not old_api_endpoint:
    with open('builder_alerts.json', 'w') as f:
      f.write(json.dumps(data, indent=1))

  ret = True

  json_data = json.dumps(data)
  logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
  gzipped_data = gzipped(json_data)

  if old_api_endpoint:
    logging.info('POST %s alerts (%s bytes compressed) to %s',
                 len(alerts), len(gzipped_data), old_api_endpoint)
    resp = requests.post(old_api_endpoint, data=gzipped_data,
                         headers={'content-encoding': 'gzip'})
    try:
      resp.raise_for_status()
    except requests.HTTPError as e:
      logging.error('POST to %s failed! %d %s, %s, %s', old_api_endpoint,
                    resp.status_code, resp.reason, resp.content, e)
      ret = False

  # Query sheriff issues and post them to the new API endpoint.
  if args.crbug_service_account:
    global issue_tracker_last_poll
    seconds_since_last_poll = (
        datetime.datetime.utcnow() - issue_tracker_last_poll).total_seconds()
    if seconds_since_last_poll > ISSUE_TRACKER_POLLING_FREQUENCY_SEC:
      issue_tracker_last_poll = datetime.datetime.utcnow()
      issues_per_tree = crbug_issues.query(args.crbug_service_account,
                                           args.use_monorail)
      for tree, issues in issues_per_tree.iteritems():
        json_data = {'alerts': issues}
        gzipped_data = gzipped(json.dumps(json_data))
        if args.api_endpoint_prefix:
          new_api_endpoint = string_helpers.slash_join(
              args.api_endpoint_prefix, 'api/v1/alerts', tree)
          logging.info('POST %s alerts (%s bytes compressed) to %s',
              len(issues), len(gzipped_data), new_api_endpoint)
          resp = requests.post(new_api_endpoint, data=gzipped_data,
                               headers={'content-encoding': 'gzip'})
          try:
            resp.raise_for_status()
          except requests.HTTPError:
            logging.exception('POST to %s failed! %d %s, %s', new_api_endpoint,
                              resp.status_code, resp.reason, resp.content)
            ret = False
        else:
          with open('builder_alerts_%s.json' % tree, 'w') as f:
            f.write(json.dumps(json_data, indent=1))
  else:
    logging.error(
        '--crbug-service-account was not specified, can not get crbug issues')
    ret = False

  return ret
Exemplo n.º 3
0
 def test_merge_by_range(self):
     groups = json.loads(self.MERGE_BY_RANGE_JSON)
     merged = analysis.merge_by_range(groups)
     self.assertEqual(len(merged), 1)
     self.assertEqual(merged[0]['sort_key'], 'dromaeo.')
     self.assertEqual(analysis.merge_by_range([]), [])
Exemplo n.º 4
0
 def test_merge_by_range(self):
   groups = json.loads(self.MERGE_BY_RANGE_JSON)
   merged = analysis.merge_by_range(groups)
   self.assertEqual(len(merged), 1)
   self.assertEqual(merged[0]['sort_key'], 'dromaeo.')
   self.assertEqual(analysis.merge_by_range([]), [])
Exemplo n.º 5
0
def inner_loop(args):
  if not args.data_url:
    logging.warn('No /data url passed, will write to builder_alerts.json')

  if args.use_cache:
    requests_cache.install_cache('failure_stats')
  else:
    requests_cache.install_cache(backend='memory')

  # FIXME: gatekeeper_config should find gatekeeper.json for us.
  gatekeeper_path = os.path.abspath(args.gatekeeper)
  logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
  gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)

  gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
  logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)
  gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
      gatekeeper_trees_path)

  master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
  start_time = datetime.datetime.utcnow()

  cache = buildbot.DiskCache(CACHE_PATH)

  old_alerts = {}
  if args.data_url:
    try:
      old_alerts_raw = requests.get(args.data_url[0]).json()
    except ValueError:
      logging.debug('No old alerts found.')
    else:
      # internal-alerts will have a redirect instead of alerts if you're
      # signed in.
      if 'alerts' in old_alerts_raw:
        for alert in old_alerts_raw['alerts']:
          master = alert['master_url']
          builder = alert['builder_name']
          step = alert['step_name']
          reason = alert['reason']
          alert_key = alert_builder.generate_alert_key(
              master, builder, step, reason)

          if alert_key in old_alerts:
            logging.critical(
                'Incorrectly overwriting an alert reason from the'
                ' old alert data. master: %s, builder: %s, step: %s, reason:'
                ' %s' % (master, builder, step, reason))

          old_alerts[alert_key] = alert

  latest_builder_info = {}
  stale_builder_alerts = []
  missing_masters = []
  alerts = []
  suspected_cls = []

  pool = multiprocessing.Pool(processes=args.processes)
  master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
                                     args.jobs), master_urls)
  pool.close()
  pool.join()

  for data in master_datas:
    # TODO(ojan): We should put an alert in the JSON for this master so
    # we can show that the master is down in the sheriff-o-matic UI.
    if not data[0]:
      missing_masters.extend([data[3]])
      continue
    alerts.extend(data[0])
    latest_builder_info.update(data[1])
    stale_builder_alerts.extend(data[2])

  logging.info('Fetch took: %s seconds.',
               (datetime.datetime.utcnow() - start_time).total_seconds())

  alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
                                                    gatekeeper_trees)
  stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
      stale_builder_alerts, gatekeeper, gatekeeper_trees)

  alerts = analysis.assign_keys(alerts)
  reason_groups = analysis.group_by_reason(alerts)
  range_groups = analysis.merge_by_range(reason_groups)

  if args.findit_api_url and alerts:
    suspected_cls = query_findit(args.findit_api_url, alerts)

  data = {
      'alerts': alerts,
      'suspected_cls': suspected_cls,
      'reason_groups': reason_groups,
      'range_groups': range_groups,
      'latest_builder_info': latest_builder_info,
      'stale_builder_alerts': stale_builder_alerts,
      'missing_masters': missing_masters,
  }

  if not args.data_url:
    with open('builder_alerts.json', 'w') as f:
      f.write(json.dumps(data, indent=1))

  ret = True

  json_data = json.dumps(data)
  logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
  s = cStringIO.StringIO()
  with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
    g.write(json_data)
  gzipped_data = s.getvalue()

  for url in args.data_url:
    logging.info('POST %s alerts (%s bytes compressed) to %s',
                 len(alerts), len(gzipped_data), url)
    resp = requests.post(url, data=gzipped_data,
                         headers={'content-encoding': 'gzip'})
    try:
      resp.raise_for_status()
    except requests.HTTPError as e:
      logging.error('POST to %s failed! %d %s, %s, %s',
                    url, resp.status_code, resp.reason, resp.content, e)
      ret = False

  return ret