예제 #1
0
 def __init__(self, *args, **kwargs):
   optparse.OptionParser.__init__(
       self, *args, prog='depot-tools-auth', version=__version__, **kwargs)
   self.add_option(
       '-v', '--verbose', action='count', default=0,
       help='Use 2 times for more debugging info')
   auth.add_auth_options(self, auth.make_auth_config(use_oauth2=True))
예제 #2
0
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument("-v", "--verbose", action="store_true")
    subparsers = parser.add_subparsers(dest="command")
    put_parser = subparsers.add_parser("put")
    put_parser.add_argument(
        "-b",
        "--bucket",
        help=(
            "The bucket to schedule the build on. Typically the master name, e.g." " master.tryserver.chromium.linux."
        ),
        required=True,
    )
    put_parser.add_argument("-c", "--changes", help="A flie to load a JSON list of changes dicts from.")
    put_parser.add_argument("-n", "--builder-name", help="The builder to schedule the build on.", required=True)
    put_parser.add_argument("-p", "--properties", help="A file to load a JSON dict of properties from.")
    args = parser.parse_args()
    # TODO(smut): When more commands are implemented, refactor this.
    assert args.command == "put"

    changes = []
    if args.changes:
        try:
            with open(args.changes) as fp:
                changes.extend(json.load(fp))
        except (TypeError, ValueError):
            sys.stderr.write("%s contained invalid JSON list.\n" % args.changes)
            raise

    properties = {}
    if args.properties:
        try:
            with open(args.properties) as fp:
                properties.update(json.load(fp))
        except (TypeError, ValueError):
            sys.stderr.write("%s contained invalid JSON dict.\n" % args.properties)
            raise

    authenticator = auth.get_authenticator_for_host(BUILDBUCKET_URL, auth.make_auth_config(use_oauth2=True))
    http = authenticator.authorize(httplib2.Http())
    http.force_exception_to_status_code = True
    response, content = http.request(
        PUT_BUILD_URL,
        "PUT",
        body=json.dumps(
            {
                "bucket": args.bucket,
                "parameters_json": json.dumps(
                    {"builder_name": args.builder_name, "changes": changes, "properties": properties}
                ),
            }
        ),
        headers={"Content-Type": "application/json"},
    )

    if args.verbose:
        print content

    return response.status != 200
예제 #3
0
 def __init__(self, *args, **kwargs):
     optparse.OptionParser.__init__(self,
                                    *args,
                                    prog='depot-tools-auth',
                                    version=__version__,
                                    **kwargs)
     self.add_option('-v',
                     '--verbose',
                     action='count',
                     default=0,
                     help='Use 2 times for more debugging info')
     auth.add_auth_options(self, auth.make_auth_config(use_oauth2=True))
예제 #4
0
def main(argv):
  parser = argparse.ArgumentParser()
  parser.add_argument(
    '-v',
    '--verbose',
    action='store_true',
  )
  subparsers = parser.add_subparsers(dest='command')
  get_parser = subparsers.add_parser('get')
  get_parser.add_argument(
    '--id',
    help='The ID of the build to get the status of.',
    required=True,
  )
  put_parser = subparsers.add_parser('put')
  put_parser.add_argument(
    '-b',
    '--bucket',
    help=(
      'The bucket to schedule the build on. Typically the master name, e.g.'
      ' master.tryserver.chromium.linux.'
    ),
    required=True,
  )
  put_parser.add_argument(
    '-c',
    '--changes',
    help='A flie to load a JSON list of changes dicts from.',
  )
  put_parser.add_argument(
    '-n',
    '--builder-name',
    help='The builder to schedule the build on.',
    required=True,
  )
  put_parser.add_argument(
    '-p',
    '--properties',
    help='A file to load a JSON dict of properties from.',
  )
  args = parser.parse_args()

  body = None

  if args.command == 'get':
    method = 'GET'
    url = '%s/%s' % (BUILDBUCKET_API_URL, args.id)
  elif args.command == 'put':
    changes = []
    if args.changes:
      try:
        with open(args.changes) as fp:
          changes.extend(json.load(fp))
      except (TypeError, ValueError):
        sys.stderr.write('%s contained invalid JSON list.\n' % args.changes)
        raise

    properties = {}
    if args.properties:
      try:
        with open(args.properties) as fp:
          properties.update(json.load(fp))
      except (TypeError, ValueError):
        sys.stderr.write('%s contained invalid JSON dict.\n' % args.properties)
        raise

    body = json.dumps({
      'bucket': args.bucket,
      'parameters_json': json.dumps({
        'builder_name': args.builder_name,
        'changes': changes,
        'properties': properties,
      }),
    })
    method = 'PUT'
    url = BUILDBUCKET_API_URL

  authenticator = auth.get_authenticator_for_host(
    BUILDBUCKET_URL,
    auth.make_auth_config(use_oauth2=True),
  )
  http = authenticator.authorize(httplib2.Http())
  http.force_exception_to_status_code = True
  response, content = http.request(
    url,
    method,
    body=body,
    headers={'Content-Type': 'application/json'},
  )

  if args.verbose:
    print content

  return response.status != 200
예제 #5
0
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
    )
    subparsers = parser.add_subparsers(dest='command')
    put_parser = subparsers.add_parser('put')
    put_parser.add_argument(
        '-b',
        '--bucket',
        help=
        ('The bucket to schedule the build on. Typically the master name, e.g.'
         ' master.tryserver.chromium.linux.'),
        required=True,
    )
    put_parser.add_argument(
        '-c',
        '--changes',
        help='A flie to load a JSON list of changes dicts from.',
    )
    put_parser.add_argument(
        '-n',
        '--builder-name',
        help='The builder to schedule the build on.',
        required=True,
    )
    put_parser.add_argument(
        '-p',
        '--properties',
        help='A file to load a JSON dict of properties from.',
    )
    args = parser.parse_args()
    # TODO(smut): When more commands are implemented, refactor this.
    assert args.command == 'put'

    changes = []
    if args.changes:
        try:
            with open(args.changes) as fp:
                changes.extend(json.load(fp))
        except (TypeError, ValueError):
            sys.stderr.write('%s contained invalid JSON list.\n' %
                             args.changes)
            raise

    properties = {}
    if args.properties:
        try:
            with open(args.properties) as fp:
                properties.update(json.load(fp))
        except (TypeError, ValueError):
            sys.stderr.write('%s contained invalid JSON dict.\n' %
                             args.properties)
            raise

    authenticator = auth.get_authenticator_for_host(
        BUILDBUCKET_URL,
        auth.make_auth_config(use_oauth2=True),
    )
    http = authenticator.authorize(httplib2.Http())
    http.force_exception_to_status_code = True
    response, content = http.request(
        PUT_BUILD_URL,
        'PUT',
        body=json.dumps({
            'bucket':
            args.bucket,
            'parameters_json':
            json.dumps({
                'builder_name': args.builder_name,
                'changes': changes,
                'properties': properties,
            }),
        }),
        headers={'Content-Type': 'application/json'},
    )

    if args.verbose:
        print content

    return response.status != 200
def CheckChangedLUCIConfigs(input_api, output_api):
  import collections
  import base64
  import json
  import urllib2

  import auth
  import git_cl

  LUCI_CONFIG_HOST_NAME = 'luci-config.appspot.com'

  cl = git_cl.Changelist()
  remote, remote_branch = cl.GetRemoteBranch()
  if remote_branch.startswith('refs/remotes/%s/' % remote):
    remote_branch = remote_branch.replace(
        'refs/remotes/%s/' % remote, 'refs/heads/', 1)
  if remote_branch.startswith('refs/remotes/branch-heads/'):
    remote_branch = remote_branch.replace(
        'refs/remotes/branch-heads/', 'refs/branch-heads/', 1)

  remote_host_url = cl.GetRemoteUrl()
  if not remote_host_url:
    return [output_api.PresubmitError(
        'Remote host url for git has not been defined')]
  remote_host_url = remote_host_url.rstrip('/')
  if remote_host_url.endswith('.git'):
    remote_host_url = remote_host_url[:-len('.git')]

  # authentication
  try:
    authenticator = auth.get_authenticator_for_host(
        LUCI_CONFIG_HOST_NAME, auth.make_auth_config())
    acc_tkn = authenticator.get_access_token()
  except auth.AuthenticationError as e:
    return [output_api.PresubmitError(
        'Error in authenticating user.', long_text=str(e))]

  def request(endpoint, body=None):
    api_url = ('https://%s/_ah/api/config/v1/%s'
               % (LUCI_CONFIG_HOST_NAME, endpoint))
    req = urllib2.Request(api_url)
    req.add_header('Authorization', 'Bearer %s' % acc_tkn.token)
    if body is not None:
      req.add_header('Content-Type', 'application/json')
      req.add_data(json.dumps(body))
    return json.load(urllib2.urlopen(req))

  try:
    config_sets = request('config-sets').get('config_sets')
  except urllib2.HTTPError as e:
    return [output_api.PresubmitError(
        'Config set request to luci-config failed', long_text=str(e))]
  if not config_sets:
    return [output_api.PresubmitWarning('No config_sets were returned')]
  loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch)
  dir_to_config_set = {
    '%s/' % cs['location'][len(loc_pref):].rstrip('/'): cs['config_set']
    for cs in config_sets
    if cs['location'].startswith(loc_pref) or
    ('%s/' % cs['location']) == loc_pref
  }
  cs_to_files = collections.defaultdict(list)
  for f in input_api.AffectedFiles():
    # windows
    file_path = f.LocalPath().replace(_os.sep, '/')
    for dr, cs in dir_to_config_set.iteritems():
      if dr == '/' or file_path.startswith(dr):
        cs_to_files[cs].append({
          'path': file_path[len(dr):] if dr != '/' else file_path,
          'content': base64.b64encode(
              '\n'.join(f.NewContents()).encode('utf-8'))
        })
  outputs = []
  for cs, f in cs_to_files.iteritems():
    try:
      # TODO(myjang): parallelize
      res = request(
          'validate-config', body={'config_set': cs, 'files': f})
    except urllib2.HTTPError as e:
      return [output_api.PresubmitError(
          'Validation request to luci-config failed', long_text=str(e))]
    for msg in res.get('messages', []):
      sev = msg['severity']
      if sev == 'WARNING':
        out_f = output_api.PresubmitPromptWarning
      elif sev == 'ERROR' or sev == 'CRITICAL':
        out_f = output_api.PresubmitError
      else:
        out_f = output_api.PresubmitNotifyResult
      outputs.append(out_f('Config validation: %s' % msg['text']))
  return outputs
예제 #7
0
 def __init__(self, *args, **kwargs):
     optparse.OptionParser.__init__(self, *args, prog="depot-tools-auth", version=__version__, **kwargs)
     self.add_option("-v", "--verbose", action="count", default=0, help="Use 2 times for more debugging info")
     auth.add_auth_options(self, auth.make_auth_config(use_oauth2=True))
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
    )
    subparsers = parser.add_subparsers(dest='command')
    get_parser = subparsers.add_parser('get')
    get_parser.add_argument(
        '--id',
        help='The ID of the build to get the status of.',
        required=True,
    )
    put_parser = subparsers.add_parser('put')
    put_parser.add_argument(
        '-b',
        '--bucket',
        help=
        ('The bucket to schedule the build on. Typically the master name, e.g.'
         ' master.tryserver.chromium.linux.'),
        required=True,
    )
    put_parser.add_argument(
        '-c',
        '--changes',
        help='A flie to load a JSON list of changes dicts from.',
    )
    put_parser.add_argument(
        '-n',
        '--builder-name',
        help='The builder to schedule the build on.',
        required=True,
    )
    put_parser.add_argument(
        '-p',
        '--properties',
        help=
        ('A file to load a JSON dict of properties from. Use "-" to pipe JSON '
         'from another command.'),
    )
    args = parser.parse_args()

    body = None

    if args.command == 'get':
        method = 'GET'
        url = '%s/%s' % (BUILDBUCKET_API_URL, args.id)
    elif args.command == 'put':
        changes = []
        if args.changes:
            try:
                with open(args.changes) as fp:
                    changes.extend(json.load(fp))
            except (TypeError, ValueError):
                sys.stderr.write('%s contained invalid JSON list.\n' %
                                 args.changes)
                raise

        properties = {}
        if args.properties:
            try:
                # Allow using pipes to stream properties from another command, e.g.
                #   echo '{"foo": "bar", "baz": 42}' | buildbucket.py -p -
                if args.properties == '-':
                    properties.update(json.load(sys.stdin))
                else:
                    with open(args.properties) as fp:
                        properties.update(json.load(fp))
            except (TypeError, ValueError):
                sys.stderr.write('%s contained invalid JSON dict.\n' %
                                 args.properties)
                raise

        body = json.dumps({
            'bucket':
            args.bucket,
            'parameters_json':
            json.dumps({
                'builder_name': args.builder_name,
                'changes': changes,
                'properties': properties,
            }),
        })
        method = 'PUT'
        url = BUILDBUCKET_API_URL

    authenticator = auth.get_authenticator_for_host(
        BUILDBUCKET_URL,
        auth.make_auth_config(use_oauth2=True),
    )
    http = authenticator.authorize(httplib2.Http())
    http.force_exception_to_status_code = True
    response, content = http.request(
        url,
        method,
        body=body,
        headers={'Content-Type': 'application/json'},
    )

    if args.verbose:
        print content

    build_url = json.loads(content).get('build', {}).get('url')
    if build_url:
        print 'Build triggered on: %s' % build_url

    return response.status != 200
def CheckChangedLUCIConfigs(input_api, output_api):
  import collections
  import base64
  import json
  import logging
  import urllib2

  import auth
  import git_cl

  LUCI_CONFIG_HOST_NAME = 'luci-config.appspot.com'

  cl = git_cl.Changelist()
  if input_api.change.issue and input_api.gerrit:
    remote_branch = input_api.gerrit.GetDestRef(input_api.change.issue)
  else:
    remote, remote_branch = cl.GetRemoteBranch()
    if remote_branch.startswith('refs/remotes/%s/' % remote):
      remote_branch = remote_branch.replace(
          'refs/remotes/%s/' % remote, 'refs/heads/', 1)
    if remote_branch.startswith('refs/remotes/branch-heads/'):
      remote_branch = remote_branch.replace(
          'refs/remotes/branch-heads/', 'refs/branch-heads/', 1)

  remote_host_url = cl.GetRemoteUrl()
  if not remote_host_url:
    return [output_api.PresubmitError(
        'Remote host url for git has not been defined')]
  remote_host_url = remote_host_url.rstrip('/')
  if remote_host_url.endswith('.git'):
    remote_host_url = remote_host_url[:-len('.git')]

  # authentication
  try:
    authenticator = auth.get_authenticator_for_host(
        LUCI_CONFIG_HOST_NAME, auth.make_auth_config())
    acc_tkn = authenticator.get_access_token()
  except auth.AuthenticationError as e:
    return [output_api.PresubmitError(
        'Error in authenticating user.', long_text=str(e))]

  def request(endpoint, body=None):
    api_url = ('https://%s/_ah/api/config/v1/%s'
               % (LUCI_CONFIG_HOST_NAME, endpoint))
    req = urllib2.Request(api_url)
    req.add_header('Authorization', 'Bearer %s' % acc_tkn.token)
    if body is not None:
      req.add_header('Content-Type', 'application/json')
      req.add_data(json.dumps(body))
    return json.load(urllib2.urlopen(req))

  try:
    config_sets = request('config-sets').get('config_sets')
  except urllib2.HTTPError as e:
    return [output_api.PresubmitError(
        'Config set request to luci-config failed', long_text=str(e))]
  if not config_sets:
    return [output_api.PresubmitWarning('No config_sets were returned')]
  loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch)
  logging.debug('Derived location prefix: %s', loc_pref)
  dir_to_config_set = {
    '%s/' % cs['location'][len(loc_pref):].rstrip('/'): cs['config_set']
    for cs in config_sets
    if cs['location'].startswith(loc_pref) or
    ('%s/' % cs['location']) == loc_pref
  }
  cs_to_files = collections.defaultdict(list)
  for f in input_api.AffectedFiles():
    # windows
    file_path = f.LocalPath().replace(_os.sep, '/')
    logging.debug('Affected file path: %s', file_path)
    for dr, cs in dir_to_config_set.iteritems():
      if dr == '/' or file_path.startswith(dr):
        cs_to_files[cs].append({
          'path': file_path[len(dr):] if dr != '/' else file_path,
          'content': base64.b64encode(
              '\n'.join(f.NewContents()).encode('utf-8'))
        })
  outputs = []
  for cs, f in cs_to_files.iteritems():
    try:
      # TODO(myjang): parallelize
      res = request(
          'validate-config', body={'config_set': cs, 'files': f})
    except urllib2.HTTPError as e:
      return [output_api.PresubmitError(
          'Validation request to luci-config failed', long_text=str(e))]
    for msg in res.get('messages', []):
      sev = msg['severity']
      if sev == 'WARNING':
        out_f = output_api.PresubmitPromptWarning
      elif sev == 'ERROR' or sev == 'CRITICAL':
        out_f = output_api.PresubmitError
      else:
        out_f = output_api.PresubmitNotifyResult
      outputs.append(out_f('Config validation: %s' % msg['text']))
  return outputs