Ejemplo n.º 1
0
    def project_hosting_issue_search(self, instance):
        auth_config = auth.extract_auth_config_from_options(self.options)
        authenticator = auth.get_authenticator_for_host(
            'bugs.chromium.org', auth_config)
        http = authenticator.authorize(httplib2.Http())
        url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects'
               '/%s/issues') % instance['name']
        epoch = datetime.utcfromtimestamp(0)
        user_str = '*****@*****.**' % self.user

        query_data = urllib.urlencode({
            'maxResults':
            10000,
            'q':
            user_str,
            'publishedMax':
            '%d' % (self.modified_before - epoch).total_seconds(),
            'updatedMin':
            '%d' % (self.modified_after - epoch).total_seconds(),
        })
        url = url + '?' + query_data
        _, body = http.request(url)
        content = json.loads(body)
        if not content:
            logging.error('Unable to parse %s response from projecthosting.',
                          instance['name'])
            return []

        issues = []
        if 'items' in content:
            items = content['items']
            for item in items:
                issue = {
                    'header':
                    item['title'],
                    'created':
                    dateutil.parser.parse(item['published']),
                    'modified':
                    dateutil.parser.parse(item['updated']),
                    'author':
                    item['author']['name'],
                    'url':
                    'https://code.google.com/p/%s/issues/detail?id=%s' %
                    (instance['name'], item['id']),
                    'comments': [],
                    'status':
                    item['status'],
                }
                if 'shorturl' in instance:
                    issue['url'] = 'http://%s/%d' % (instance['shorturl'],
                                                     item['id'])

                if 'owner' in item:
                    issue['owner'] = item['owner']['name']
                else:
                    issue['owner'] = 'None'
                if issue['owner'] == user_str or issue['author'] == user_str:
                    issues.append(issue)

        return issues
Ejemplo n.º 2
0
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument("-v", "--verbose", action="store_true")
    subparsers = parser.add_subparsers(dest="command")
    put_parser = subparsers.add_parser("put")
    put_parser.add_argument(
        "-b",
        "--bucket",
        help=(
            "The bucket to schedule the build on. Typically the master name, e.g." " master.tryserver.chromium.linux."
        ),
        required=True,
    )
    put_parser.add_argument("-c", "--changes", help="A flie to load a JSON list of changes dicts from.")
    put_parser.add_argument("-n", "--builder-name", help="The builder to schedule the build on.", required=True)
    put_parser.add_argument("-p", "--properties", help="A file to load a JSON dict of properties from.")
    args = parser.parse_args()
    # TODO(smut): When more commands are implemented, refactor this.
    assert args.command == "put"

    changes = []
    if args.changes:
        try:
            with open(args.changes) as fp:
                changes.extend(json.load(fp))
        except (TypeError, ValueError):
            sys.stderr.write("%s contained invalid JSON list.\n" % args.changes)
            raise

    properties = {}
    if args.properties:
        try:
            with open(args.properties) as fp:
                properties.update(json.load(fp))
        except (TypeError, ValueError):
            sys.stderr.write("%s contained invalid JSON dict.\n" % args.properties)
            raise

    authenticator = auth.get_authenticator_for_host(BUILDBUCKET_URL, auth.make_auth_config(use_oauth2=True))
    http = authenticator.authorize(httplib2.Http())
    http.force_exception_to_status_code = True
    response, content = http.request(
        PUT_BUILD_URL,
        "PUT",
        body=json.dumps(
            {
                "bucket": args.bucket,
                "parameters_json": json.dumps(
                    {"builder_name": args.builder_name, "changes": changes, "properties": properties}
                ),
            }
        ),
        headers={"Content-Type": "application/json"},
    )

    if args.verbose:
        print content

    return response.status != 200
Ejemplo n.º 3
0
 def monorail_get_auth_http(self):
     auth_config = auth.extract_auth_config_from_options(self.options)
     authenticator = auth.get_authenticator_for_host(
         'bugs.chromium.org', auth_config)
     # Manually use a long timeout (10m); for some users who have a
     # long history on the issue tracker, whatever the default timeout
     # is is reached.
     return authenticator.authorize(httplib2.Http(timeout=600))
Ejemplo n.º 4
0
 def monorail_get_auth_http(self):
   auth_config = auth.extract_auth_config_from_options(self.options)
   authenticator = auth.get_authenticator_for_host(
       'bugs.chromium.org', auth_config)
   # Manually use a long timeout (10m); for some users who have a
   # long history on the issue tracker, whatever the default timeout
   # is is reached.
   return authenticator.authorize(httplib2.Http(timeout=600))
Ejemplo n.º 5
0
 def parse_args(self, args=None, values=None):
     """Parses options and returns (hostname, auth.Authenticator object)."""
     options, args = optparse.OptionParser.parse_args(self, args, values)
     levels = [logging.WARNING, logging.INFO, logging.DEBUG]
     logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
     auth_config = auth.extract_auth_config_from_options(options)
     if len(args) != 1:
         self.error("Expecting single argument (hostname).")
     if not auth_config.use_oauth2:
         self.error("This command is only usable with OAuth2 authentication")
     return args[0], auth.get_authenticator_for_host(args[0], auth_config)
Ejemplo n.º 6
0
    def project_hosting_issue_search(self, instance):
        auth_config = auth.extract_auth_config_from_options(self.options)
        authenticator = auth.get_authenticator_for_host(
            "code.google.com", auth_config)
        http = authenticator.authorize(httplib2.Http())
        url = "https://www.googleapis.com/projecthosting/v2/projects/%s/issues" % (
            instance["name"])
        epoch = datetime.utcfromtimestamp(0)
        user_str = '*****@*****.**' % self.user

        query_data = urllib.urlencode({
            'maxResults':
            10000,
            'q':
            user_str,
            'publishedMax':
            '%d' % (self.modified_before - epoch).total_seconds(),
            'updatedMin':
            '%d' % (self.modified_after - epoch).total_seconds(),
        })
        url = url + '?' + query_data
        _, body = http.request(url)
        content = json.loads(body)
        if not content:
            print "Unable to parse %s response from projecthosting." % (
                instance["name"])
            return []

        issues = []
        if 'items' in content:
            items = content['items']
            for item in items:
                issue = {
                    "header":
                    item["title"],
                    "created":
                    item["published"],
                    "modified":
                    item["updated"],
                    "author":
                    item["author"]["name"],
                    "url":
                    "https://code.google.com/p/%s/issues/detail?id=%s" %
                    (instance["name"], item["id"]),
                    "comments": []
                }
                if 'owner' in item:
                    issue['owner'] = item['owner']['name']
                else:
                    issue['owner'] = 'None'
                if issue['owner'] == user_str or issue['author'] == user_str:
                    issues.append(issue)

        return issues
Ejemplo n.º 7
0
 def parse_args(self, args=None, values=None):
     """Parses options and returns (hostname, auth.Authenticator object)."""
     options, args = optparse.OptionParser.parse_args(self, args, values)
     levels = [logging.WARNING, logging.INFO, logging.DEBUG]
     logging.basicConfig(level=levels[min(options.verbose,
                                          len(levels) - 1)])
     auth_config = auth.extract_auth_config_from_options(options)
     if len(args) != 1:
         self.error('Expecting single argument (hostname).')
     if not auth_config.use_oauth2:
         self.error(
             'This command is only usable with OAuth2 authentication')
     return args[0], auth.get_authenticator_for_host(args[0], auth_config)
Ejemplo n.º 8
0
  def project_hosting_issue_search(self, instance):
    auth_config = auth.extract_auth_config_from_options(self.options)
    authenticator = auth.get_authenticator_for_host(
        "code.google.com", auth_config)
    http = authenticator.authorize(httplib2.Http())
    url = "https://www.googleapis.com/projecthosting/v2/projects/%s/issues" % (
       instance["name"])
    epoch = datetime.utcfromtimestamp(0)
    user_str = '*****@*****.**' % self.user

    query_data = urllib.urlencode({
      'maxResults': 10000,
      'q': user_str,
      'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(),
      'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(),
    })
    url = url + '?' + query_data
    _, body = http.request(url)
    content = json.loads(body)
    if not content:
      print "Unable to parse %s response from projecthosting." % (
          instance["name"])
      return []

    issues = []
    if 'items' in content:
      items = content['items']
      for item in items:
        issue = {
          "header": item["title"],
          "created": item["published"],
          "modified": item["updated"],
          "author": item["author"]["name"],
          "url": "https://code.google.com/p/%s/issues/detail?id=%s" % (
              instance["name"], item["id"]),
          "comments": []
        }
        if 'owner' in item:
          issue['owner'] = item['owner']['name']
        else:
          issue['owner'] = 'None'
        if issue['owner'] == user_str or issue['author'] == user_str:
          issues.append(issue)

    return issues
Ejemplo n.º 9
0
def main(argv):
  parser = argparse.ArgumentParser()
  parser.add_argument(
    '-v',
    '--verbose',
    action='store_true',
  )
  subparsers = parser.add_subparsers(dest='command')
  get_parser = subparsers.add_parser('get')
  get_parser.add_argument(
    '--id',
    help='The ID of the build to get the status of.',
    required=True,
  )
  put_parser = subparsers.add_parser('put')
  put_parser.add_argument(
    '-b',
    '--bucket',
    help=(
      'The bucket to schedule the build on. Typically the master name, e.g.'
      ' master.tryserver.chromium.linux.'
    ),
    required=True,
  )
  put_parser.add_argument(
    '-c',
    '--changes',
    help='A flie to load a JSON list of changes dicts from.',
  )
  put_parser.add_argument(
    '-n',
    '--builder-name',
    help='The builder to schedule the build on.',
    required=True,
  )
  put_parser.add_argument(
    '-p',
    '--properties',
    help='A file to load a JSON dict of properties from.',
  )
  args = parser.parse_args()

  body = None

  if args.command == 'get':
    method = 'GET'
    url = '%s/%s' % (BUILDBUCKET_API_URL, args.id)
  elif args.command == 'put':
    changes = []
    if args.changes:
      try:
        with open(args.changes) as fp:
          changes.extend(json.load(fp))
      except (TypeError, ValueError):
        sys.stderr.write('%s contained invalid JSON list.\n' % args.changes)
        raise

    properties = {}
    if args.properties:
      try:
        with open(args.properties) as fp:
          properties.update(json.load(fp))
      except (TypeError, ValueError):
        sys.stderr.write('%s contained invalid JSON dict.\n' % args.properties)
        raise

    body = json.dumps({
      'bucket': args.bucket,
      'parameters_json': json.dumps({
        'builder_name': args.builder_name,
        'changes': changes,
        'properties': properties,
      }),
    })
    method = 'PUT'
    url = BUILDBUCKET_API_URL

  authenticator = auth.get_authenticator_for_host(
    BUILDBUCKET_URL,
    auth.make_auth_config(use_oauth2=True),
  )
  http = authenticator.authorize(httplib2.Http())
  http.force_exception_to_status_code = True
  response, content = http.request(
    url,
    method,
    body=body,
    headers={'Content-Type': 'application/json'},
  )

  if args.verbose:
    print content

  return response.status != 200
Ejemplo n.º 10
0
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
    )
    subparsers = parser.add_subparsers(dest='command')
    put_parser = subparsers.add_parser('put')
    put_parser.add_argument(
        '-b',
        '--bucket',
        help=
        ('The bucket to schedule the build on. Typically the master name, e.g.'
         ' master.tryserver.chromium.linux.'),
        required=True,
    )
    put_parser.add_argument(
        '-c',
        '--changes',
        help='A flie to load a JSON list of changes dicts from.',
    )
    put_parser.add_argument(
        '-n',
        '--builder-name',
        help='The builder to schedule the build on.',
        required=True,
    )
    put_parser.add_argument(
        '-p',
        '--properties',
        help='A file to load a JSON dict of properties from.',
    )
    args = parser.parse_args()
    # TODO(smut): When more commands are implemented, refactor this.
    assert args.command == 'put'

    changes = []
    if args.changes:
        try:
            with open(args.changes) as fp:
                changes.extend(json.load(fp))
        except (TypeError, ValueError):
            sys.stderr.write('%s contained invalid JSON list.\n' %
                             args.changes)
            raise

    properties = {}
    if args.properties:
        try:
            with open(args.properties) as fp:
                properties.update(json.load(fp))
        except (TypeError, ValueError):
            sys.stderr.write('%s contained invalid JSON dict.\n' %
                             args.properties)
            raise

    authenticator = auth.get_authenticator_for_host(
        BUILDBUCKET_URL,
        auth.make_auth_config(use_oauth2=True),
    )
    http = authenticator.authorize(httplib2.Http())
    http.force_exception_to_status_code = True
    response, content = http.request(
        PUT_BUILD_URL,
        'PUT',
        body=json.dumps({
            'bucket':
            args.bucket,
            'parameters_json':
            json.dumps({
                'builder_name': args.builder_name,
                'changes': changes,
                'properties': properties,
            }),
        }),
        headers={'Content-Type': 'application/json'},
    )

    if args.verbose:
        print content

    return response.status != 200
Ejemplo n.º 11
0
 def monorail_get_auth_http(self):
     auth_config = auth.extract_auth_config_from_options(self.options)
     authenticator = auth.get_authenticator_for_host(
         'bugs.chromium.org', auth_config)
     return authenticator.authorize(httplib2.Http())
Ejemplo n.º 12
0
 def has_cookie(instance):
     auth_config = auth.extract_auth_config_from_options(self.options)
     a = auth.get_authenticator_for_host(instance['url'], auth_config)
     return a.has_cached_credentials()
def CheckChangedLUCIConfigs(input_api, output_api):
  import collections
  import base64
  import json
  import urllib2

  import auth
  import git_cl

  LUCI_CONFIG_HOST_NAME = 'luci-config.appspot.com'

  cl = git_cl.Changelist()
  remote, remote_branch = cl.GetRemoteBranch()
  if remote_branch.startswith('refs/remotes/%s/' % remote):
    remote_branch = remote_branch.replace(
        'refs/remotes/%s/' % remote, 'refs/heads/', 1)
  if remote_branch.startswith('refs/remotes/branch-heads/'):
    remote_branch = remote_branch.replace(
        'refs/remotes/branch-heads/', 'refs/branch-heads/', 1)

  remote_host_url = cl.GetRemoteUrl()
  if not remote_host_url:
    return [output_api.PresubmitError(
        'Remote host url for git has not been defined')]
  remote_host_url = remote_host_url.rstrip('/')
  if remote_host_url.endswith('.git'):
    remote_host_url = remote_host_url[:-len('.git')]

  # authentication
  try:
    authenticator = auth.get_authenticator_for_host(
        LUCI_CONFIG_HOST_NAME, auth.make_auth_config())
    acc_tkn = authenticator.get_access_token()
  except auth.AuthenticationError as e:
    return [output_api.PresubmitError(
        'Error in authenticating user.', long_text=str(e))]

  def request(endpoint, body=None):
    api_url = ('https://%s/_ah/api/config/v1/%s'
               % (LUCI_CONFIG_HOST_NAME, endpoint))
    req = urllib2.Request(api_url)
    req.add_header('Authorization', 'Bearer %s' % acc_tkn.token)
    if body is not None:
      req.add_header('Content-Type', 'application/json')
      req.add_data(json.dumps(body))
    return json.load(urllib2.urlopen(req))

  try:
    config_sets = request('config-sets').get('config_sets')
  except urllib2.HTTPError as e:
    return [output_api.PresubmitError(
        'Config set request to luci-config failed', long_text=str(e))]
  if not config_sets:
    return [output_api.PresubmitWarning('No config_sets were returned')]
  loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch)
  dir_to_config_set = {
    '%s/' % cs['location'][len(loc_pref):].rstrip('/'): cs['config_set']
    for cs in config_sets
    if cs['location'].startswith(loc_pref) or
    ('%s/' % cs['location']) == loc_pref
  }
  cs_to_files = collections.defaultdict(list)
  for f in input_api.AffectedFiles():
    # windows
    file_path = f.LocalPath().replace(_os.sep, '/')
    for dr, cs in dir_to_config_set.iteritems():
      if dr == '/' or file_path.startswith(dr):
        cs_to_files[cs].append({
          'path': file_path[len(dr):] if dr != '/' else file_path,
          'content': base64.b64encode(
              '\n'.join(f.NewContents()).encode('utf-8'))
        })
  outputs = []
  for cs, f in cs_to_files.iteritems():
    try:
      # TODO(myjang): parallelize
      res = request(
          'validate-config', body={'config_set': cs, 'files': f})
    except urllib2.HTTPError as e:
      return [output_api.PresubmitError(
          'Validation request to luci-config failed', long_text=str(e))]
    for msg in res.get('messages', []):
      sev = msg['severity']
      if sev == 'WARNING':
        out_f = output_api.PresubmitPromptWarning
      elif sev == 'ERROR' or sev == 'CRITICAL':
        out_f = output_api.PresubmitError
      else:
        out_f = output_api.PresubmitNotifyResult
      outputs.append(out_f('Config validation: %s' % msg['text']))
  return outputs
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
    )
    subparsers = parser.add_subparsers(dest='command')
    get_parser = subparsers.add_parser('get')
    get_parser.add_argument(
        '--id',
        help='The ID of the build to get the status of.',
        required=True,
    )
    put_parser = subparsers.add_parser('put')
    put_parser.add_argument(
        '-b',
        '--bucket',
        help=
        ('The bucket to schedule the build on. Typically the master name, e.g.'
         ' master.tryserver.chromium.linux.'),
        required=True,
    )
    put_parser.add_argument(
        '-c',
        '--changes',
        help='A flie to load a JSON list of changes dicts from.',
    )
    put_parser.add_argument(
        '-n',
        '--builder-name',
        help='The builder to schedule the build on.',
        required=True,
    )
    put_parser.add_argument(
        '-p',
        '--properties',
        help=
        ('A file to load a JSON dict of properties from. Use "-" to pipe JSON '
         'from another command.'),
    )
    args = parser.parse_args()

    body = None

    if args.command == 'get':
        method = 'GET'
        url = '%s/%s' % (BUILDBUCKET_API_URL, args.id)
    elif args.command == 'put':
        changes = []
        if args.changes:
            try:
                with open(args.changes) as fp:
                    changes.extend(json.load(fp))
            except (TypeError, ValueError):
                sys.stderr.write('%s contained invalid JSON list.\n' %
                                 args.changes)
                raise

        properties = {}
        if args.properties:
            try:
                # Allow using pipes to stream properties from another command, e.g.
                #   echo '{"foo": "bar", "baz": 42}' | buildbucket.py -p -
                if args.properties == '-':
                    properties.update(json.load(sys.stdin))
                else:
                    with open(args.properties) as fp:
                        properties.update(json.load(fp))
            except (TypeError, ValueError):
                sys.stderr.write('%s contained invalid JSON dict.\n' %
                                 args.properties)
                raise

        body = json.dumps({
            'bucket':
            args.bucket,
            'parameters_json':
            json.dumps({
                'builder_name': args.builder_name,
                'changes': changes,
                'properties': properties,
            }),
        })
        method = 'PUT'
        url = BUILDBUCKET_API_URL

    authenticator = auth.get_authenticator_for_host(
        BUILDBUCKET_URL,
        auth.make_auth_config(use_oauth2=True),
    )
    http = authenticator.authorize(httplib2.Http())
    http.force_exception_to_status_code = True
    response, content = http.request(
        url,
        method,
        body=body,
        headers={'Content-Type': 'application/json'},
    )

    if args.verbose:
        print content

    build_url = json.loads(content).get('build', {}).get('url')
    if build_url:
        print 'Build triggered on: %s' % build_url

    return response.status != 200
Ejemplo n.º 15
0
 def has_cookie(instance):
   auth_config = auth.extract_auth_config_from_options(self.options)
   a = auth.get_authenticator_for_host(instance['url'], auth_config)
   return a.has_cached_credentials()
def CheckChangedLUCIConfigs(input_api, output_api):
  import collections
  import base64
  import json
  import logging
  import urllib2

  import auth
  import git_cl

  LUCI_CONFIG_HOST_NAME = 'luci-config.appspot.com'

  cl = git_cl.Changelist()
  if input_api.change.issue and input_api.gerrit:
    remote_branch = input_api.gerrit.GetDestRef(input_api.change.issue)
  else:
    remote, remote_branch = cl.GetRemoteBranch()
    if remote_branch.startswith('refs/remotes/%s/' % remote):
      remote_branch = remote_branch.replace(
          'refs/remotes/%s/' % remote, 'refs/heads/', 1)
    if remote_branch.startswith('refs/remotes/branch-heads/'):
      remote_branch = remote_branch.replace(
          'refs/remotes/branch-heads/', 'refs/branch-heads/', 1)

  remote_host_url = cl.GetRemoteUrl()
  if not remote_host_url:
    return [output_api.PresubmitError(
        'Remote host url for git has not been defined')]
  remote_host_url = remote_host_url.rstrip('/')
  if remote_host_url.endswith('.git'):
    remote_host_url = remote_host_url[:-len('.git')]

  # authentication
  try:
    authenticator = auth.get_authenticator_for_host(
        LUCI_CONFIG_HOST_NAME, auth.make_auth_config())
    acc_tkn = authenticator.get_access_token()
  except auth.AuthenticationError as e:
    return [output_api.PresubmitError(
        'Error in authenticating user.', long_text=str(e))]

  def request(endpoint, body=None):
    api_url = ('https://%s/_ah/api/config/v1/%s'
               % (LUCI_CONFIG_HOST_NAME, endpoint))
    req = urllib2.Request(api_url)
    req.add_header('Authorization', 'Bearer %s' % acc_tkn.token)
    if body is not None:
      req.add_header('Content-Type', 'application/json')
      req.add_data(json.dumps(body))
    return json.load(urllib2.urlopen(req))

  try:
    config_sets = request('config-sets').get('config_sets')
  except urllib2.HTTPError as e:
    return [output_api.PresubmitError(
        'Config set request to luci-config failed', long_text=str(e))]
  if not config_sets:
    return [output_api.PresubmitWarning('No config_sets were returned')]
  loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch)
  logging.debug('Derived location prefix: %s', loc_pref)
  dir_to_config_set = {
    '%s/' % cs['location'][len(loc_pref):].rstrip('/'): cs['config_set']
    for cs in config_sets
    if cs['location'].startswith(loc_pref) or
    ('%s/' % cs['location']) == loc_pref
  }
  cs_to_files = collections.defaultdict(list)
  for f in input_api.AffectedFiles():
    # windows
    file_path = f.LocalPath().replace(_os.sep, '/')
    logging.debug('Affected file path: %s', file_path)
    for dr, cs in dir_to_config_set.iteritems():
      if dr == '/' or file_path.startswith(dr):
        cs_to_files[cs].append({
          'path': file_path[len(dr):] if dr != '/' else file_path,
          'content': base64.b64encode(
              '\n'.join(f.NewContents()).encode('utf-8'))
        })
  outputs = []
  for cs, f in cs_to_files.iteritems():
    try:
      # TODO(myjang): parallelize
      res = request(
          'validate-config', body={'config_set': cs, 'files': f})
    except urllib2.HTTPError as e:
      return [output_api.PresubmitError(
          'Validation request to luci-config failed', long_text=str(e))]
    for msg in res.get('messages', []):
      sev = msg['severity']
      if sev == 'WARNING':
        out_f = output_api.PresubmitPromptWarning
      elif sev == 'ERROR' or sev == 'CRITICAL':
        out_f = output_api.PresubmitError
      else:
        out_f = output_api.PresubmitNotifyResult
      outputs.append(out_f('Config validation: %s' % msg['text']))
  return outputs