def monorail_get_auth_http(self):
auth_config = auth.extract_auth_config_from_options(self.options)
authenticator = auth.get_authenticator(auth_config)
# Manually use a long timeout (10m); for some users who have a
# long history on the issue tracker, whatever the default timeout
# is is reached.
return authenticator.authorize(httplib2.Http(timeout=600))
def project_hosting_issue_search(self, instance):
auth_config = auth.extract_auth_config_from_options(self.options)
authenticator = auth.get_authenticator_for_host(
'bugs.chromium.org', auth_config)
http = authenticator.authorize(httplib2.Http())
url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects'
'/%s/issues') % instance['name']
epoch = datetime.utcfromtimestamp(0)
user_str = '*****@*****.**' % self.user
query_data = urllib.urlencode({
'maxResults':
10000,
'q':
user_str,
'publishedMax':
'%d' % (self.modified_before - epoch).total_seconds(),
'updatedMin':
'%d' % (self.modified_after - epoch).total_seconds(),
})
url = url + '?' + query_data
_, body = http.request(url)
content = json.loads(body)
if not content:
logging.error('Unable to parse %s response from projecthosting.',
instance['name'])
return []
issues = []
if 'items' in content:
items = content['items']
for item in items:
issue = {
'header':
item['title'],
'created':
dateutil.parser.parse(item['published']),
'modified':
dateutil.parser.parse(item['updated']),
'author':
item['author']['name'],
'url':
'https://code.google.com/p/%s/issues/detail?id=%s' %
(instance['name'], item['id']),
'comments': [],
'status':
item['status'],
}
if 'shorturl' in instance:
issue['url'] = 'http://%s/%d' % (instance['shorturl'],
item['id'])
if 'owner' in item:
issue['owner'] = item['owner']['name']
else:
issue['owner'] = 'None'
if issue['owner'] == user_str or issue['author'] == user_str:
issues.append(issue)
return issues
def GetConnectionObject(protocol=None):
if protocol is None:
protocol = GERRIT_PROTOCOL
if protocol in ('http', 'https'):
return httplib2.Http()
else:
raise RuntimeError("Don't know how to work with protocol '%s'" %
protocol)
def IsGoogler(server):
"""Check whether this script run inside corp network."""
try:
h = httplib2.Http()
_, content = h.request('https://' + server + '/should-upload', 'GET')
return content == 'Success'
except httplib2.HttpLib2Error:
return False
def _get_luci_context_access_token(params, now, scopes=OAUTH_SCOPE_EMAIL):
# No account, local_auth shouldn't be used.
if not params.default_account_id:
return None
if not params.secret:
raise LuciContextAuthError('local_auth: no secret')
logging.debug('local_auth: requesting an access token for account "%s"',
params.default_account_id)
http = httplib2.Http()
host = '127.0.0.1:%d' % params.rpc_port
resp, content = http.request(
uri='http://%s/rpc/LuciLocalAuthService.GetOAuthToken' % host,
method='POST',
body=json.dumps({
'account_id': params.default_account_id,
'scopes': scopes.split(' '),
'secret': params.secret,
}),
headers={'Content-Type': 'application/json'})
if resp.status != 200:
raise LuciContextAuthError(
'local_auth: Failed to grab access token from '
'LUCI context server with status %d: %r' % (resp.status, content))
try:
token = json.loads(content)
error_code = token.get('error_code')
error_message = token.get('error_message')
access_token = token.get('access_token')
expiry = token.get('expiry')
except (AttributeError, ValueError) as e:
raise LuciContextAuthError('Unexpected access token response format',
e)
if error_code:
raise LuciContextAuthError('Error %d in retrieving access token: %s',
error_code, error_message)
if not access_token:
raise LuciContextAuthError(
'No access token returned from LUCI context server')
expiry_dt = None
if expiry:
try:
expiry_dt = datetime.datetime.utcfromtimestamp(expiry)
logging.debug(
'local_auth: got an access token for '
'account "%s" that expires in %d sec',
params.default_account_id, (expiry_dt - now).total_seconds())
except (TypeError, ValueError) as e:
raise LuciContextAuthError('Invalid expiry in returned token', e)
else:
logging.debug(
'local auth: got an access token for account "%s" that does not expire',
params.default_account_id)
access_token = AccessToken(access_token, expiry_dt)
if access_token.needs_refresh(now=now):
raise LuciContextAuthError('Received access token is already expired')
return access_token
def _create_access_token(self, allow_user_interaction=False):
"""Mints and caches a new access token, launching OAuth2 dance if necessary.
Uses cached refresh token, if present. In that case user interaction is not
required and function will finish quietly. Otherwise it will launch 3-legged
OAuth2 flow, that needs user interaction.
Args:
allow_user_interaction: if True, allow interaction with the user (e.g.
reading standard input, or launching a browser).
Returns:
AccessToken.
Raises:
AuthenticationError on error or if authentication flow was interrupted.
LoginRequiredError if user interaction is required, but
allow_user_interaction is False.
"""
logging.debug('Making new access token (allow_user_interaction=%r)',
allow_user_interaction)
credentials = self._get_cached_credentials()
# 3-legged flow with (perhaps cached) refresh token.
refreshed = False
if credentials and not credentials.invalid:
try:
logging.debug('Attempting to refresh access_token')
credentials.refresh(httplib2.Http())
_log_credentials_info('refreshed token', credentials)
refreshed = True
except client.Error as err:
logging.warning(
'OAuth error during access token refresh (%s). '
'Attempting a full authentication flow.', err)
# Refresh token is missing or invalid, go through the full flow.
if not refreshed:
# Can't refresh externally provided token.
if self._external_token:
raise AuthenticationError(
'Token provided via --auth-refresh-token-json is no longer valid.'
)
if not allow_user_interaction:
logging.debug('Requesting user to login')
raise LoginRequiredError(self._token_cache_key)
logging.debug('Launching OAuth browser flow')
credentials = _run_oauth_dance(self._config, self._scopes)
_log_credentials_info('new token', credentials)
logging.info('OAuth access_token refreshed. Expires in %s.',
credentials.token_expiry - datetime.datetime.utcnow())
storage = self._get_storage()
credentials.set_store(storage)
storage.put(credentials)
return AccessToken(str(credentials.access_token),
credentials.token_expiry)
def get_token_info(self):
"""Returns a result of /oauth2/v2/tokeninfo call with token info."""
access_token = self.get_access_token()
resp, content = httplib2.Http().request(
uri='https://www.googleapis.com/oauth2/v2/tokeninfo?%s' % (
urllib.urlencode({'access_token': access_token.token})))
if resp.status == 200:
return json.loads(content)
raise AuthenticationError('Failed to fetch the token info: %r' % content)
def project_hosting_issue_search(self, instance):
auth_config = auth.extract_auth_config_from_options(self.options)
authenticator = auth.get_authenticator_for_host(
"code.google.com", auth_config)
http = authenticator.authorize(httplib2.Http())
url = "https://www.googleapis.com/projecthosting/v2/projects/%s/issues" % (
instance["name"])
epoch = datetime.utcfromtimestamp(0)
user_str = '*****@*****.**' % self.user
query_data = urllib.urlencode({
'maxResults':
10000,
'q':
user_str,
'publishedMax':
'%d' % (self.modified_before - epoch).total_seconds(),
'updatedMin':
'%d' % (self.modified_after - epoch).total_seconds(),
})
url = url + '?' + query_data
_, body = http.request(url)
content = json.loads(body)
if not content:
print "Unable to parse %s response from projecthosting." % (
instance["name"])
return []
issues = []
if 'items' in content:
items = content['items']
for item in items:
issue = {
"header":
item["title"],
"created":
item["published"],
"modified":
item["updated"],
"author":
item["author"]["name"],
"url":
"https://code.google.com/p/%s/issues/detail?id=%s" %
(instance["name"], item["id"]),
"comments": []
}
if 'owner' in item:
issue['owner'] = item['owner']['name']
else:
issue['owner'] = 'None'
if issue['owner'] == user_str or issue['author'] == user_str:
issues.append(issue)
return issues
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--server',
default='chromium-build-stats.appspot.com',
help='server to upload ninjalog file.')
parser.add_argument('--ninjalog', help='ninjalog file to upload.')
parser.add_argument('--verbose', action='store_true',
help='Enable verbose logging.')
parser.add_argument('--cmdline', required=True, nargs=argparse.REMAINDER,
help='command line args passed to ninja.')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.INFO)
else:
# Disable logging.
logging.disable(logging.CRITICAL)
if not IsGoogler(args.server):
return 0
ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
if not os.path.isfile(ninjalog):
logging.warn("ninjalog is not found in %s", ninjalog)
return 1
output = cStringIO.StringIO()
with open(ninjalog) as f:
with gzip.GzipFile(fileobj=output, mode='wb') as g:
g.write(f.read())
g.write('# end of ninja log\n')
metadata = GetMetadata(args.cmdline, ninjalog)
logging.info('send metadata: %s', metadata)
g.write(json.dumps(metadata))
h = httplib2.Http()
resp_headers, content = h.request(
'https://'+args.server+'/upload_ninja_log/', 'POST',
body=output.getvalue(), headers={'Content-Encoding': 'gzip'})
if resp_headers.status != 200:
logging.warn("unexpected status code for response: %s",
resp_headers.status)
return 1
logging.info('response header: %s', resp_headers)
logging.info('response content: %s', content)
return 0
def __init__(self,
host,
client_email,
client_private_key,
private_key_password='******',
user_agent=None,
timeout=None,
extra_headers=None):
"""Wrapper around httplib2.Http() that handles authentication.
client_email: email associated with the service account
client_private_key: encrypted private key, as a string
private_key_password: password used to decrypt the private key
"""
# Enforce https
host_parts = urlparse.urlparse(host)
if host_parts.scheme == 'https': # fine
self.host = host
elif host_parts.scheme == 'http':
upload.logging.warning('Changing protocol to https')
self.host = 'https' + host[4:]
else:
msg = 'Invalid url provided: %s' % host
upload.logging.error(msg)
raise ValueError(msg)
self.host = self.host.rstrip('/')
self.extra_headers = extra_headers or {}
if not oa2client.HAS_OPENSSL:
logging.error("No support for OpenSSL has been found, "
"OAuth2 support requires it.")
logging.error(
"Installing pyopenssl will probably solve this issue.")
raise RuntimeError('No OpenSSL support')
self.creds = oa2client.SignedJwtAssertionCredentials(
client_email,
client_private_key,
'https://www.googleapis.com/auth/userinfo.email',
private_key_password=private_key_password,
user_agent=user_agent)
self._http = self.creds.authorize(httplib2.Http(timeout=timeout))
def logout(self):
"""Revokes the refresh token and deletes it from the cache.
Returns True if had some credentials cached.
"""
with self._lock:
self._access_token = None
storage = self._get_storage()
credentials = storage.get()
had_creds = bool(credentials)
if credentials and credentials.refresh_token and credentials.revoke_uri:
try:
credentials.revoke(httplib2.Http())
except client.TokenRevokeError as e:
logging.warning('Failed to revoke refresh token: %s', e)
storage.delete()
return had_creds
def authenticated_http_request(service_account, *args, **kwargs):
"""Sends an OAuth2-authenticated HTTP request.
Args:
service_account: Service account to use. For GCE, the name of the service
account, otherwise the path to the service account JSON file.
Raises:
AuthenticatedHttpRequestFailure
"""
scopes = kwargs.pop('scopes', [])
kwargs['headers'] = kwargs.get('headers', {}).copy()
http = httplib2.Http(ca_certs=tools.get_cacerts_bundle())
# Authorize the request. In general, we need to produce an OAuth2 bearer token
# using a service account JSON file. However on GCE there is a shortcut: it
# can fetch the current bearer token right from the instance metadata without
# the need for the oauth2client.client library.
if platforms.is_gce():
try:
gce_bearer_token, _ = platforms.gce.oauth2_access_token_with_expiration(
account=service_account)
except (IOError, urllib2.HTTPError) as e:
raise AuthenticatedHttpRequestFailure(e)
kwargs['headers']['Authorization'] = 'Bearer %s' % gce_bearer_token
else:
try:
oauth2client = get_oauth2_client(service_account, scopes)
except (IOError, OSError, ValueError) as e:
raise AuthenticatedHttpRequestFailure(e)
http = oauth2client.authorize(http)
try:
return http.request(*args, **kwargs)
except client.Error as e:
raise AuthenticatedHttpRequestFailure(e)
def _get_luci_context_access_token(env, now):
ctx_path = env.get('LUCI_CONTEXT')
if not ctx_path:
return None
ctx_path = ctx_path.decode(sys.getfilesystemencoding())
logging.debug('Loading LUCI_CONTEXT: %r', ctx_path)
def authErr(msg, *args):
error_msg = msg % args
ex = sys.exc_info()[1]
if not ex:
logging.error(error_msg)
raise LuciContextAuthError(error_msg)
logging.exception(error_msg)
raise LuciContextAuthError('%s: %s' % (error_msg, ex))
try:
loaded = _load_luci_context(ctx_path)
except (OSError, IOError, ValueError):
authErr('Failed to open, read or decode LUCI_CONTEXT')
try:
local_auth = loaded.get('local_auth')
except AttributeError:
authErr('LUCI_CONTEXT not in proper format')
# failed to grab local_auth from LUCI context
if not local_auth:
logging.debug('local_auth: no local auth found')
return None
try:
account_id = local_auth.get('default_account_id')
secret = local_auth.get('secret')
rpc_port = int(local_auth.get('rpc_port'))
except (AttributeError, ValueError):
authErr('local_auth: unexpected local auth format')
if not secret:
authErr('local_auth: no secret returned')
# if account_id not specified, LUCI_CONTEXT should not be picked up
if not account_id:
return None
logging.debug('local_auth: requesting an access token for account "%s"',
account_id)
http = httplib2.Http()
host = '127.0.0.1:%d' % rpc_port
resp, content = http.request(
uri='http://%s/rpc/LuciLocalAuthService.GetOAuthToken' % host,
method='POST',
body=json.dumps({
'account_id': account_id,
'scopes': OAUTH_SCOPES.split(' '),
'secret': secret,
}),
headers={'Content-Type': 'application/json'})
if resp.status != 200:
err = ('local_auth: Failed to grab access token from '
'LUCI context server with status %d: %r')
authErr(err, resp.status, content)
try:
token = json.loads(content)
error_code = token.get('error_code')
error_message = token.get('error_message')
access_token = token.get('access_token')
expiry = token.get('expiry')
except (AttributeError, ValueError):
authErr('local_auth: Unexpected access token response format')
if error_code:
authErr('local_auth: Error %d in retrieving access token: %s',
error_code, error_message)
if not access_token:
authErr(
'local_auth: No access token returned from LUCI context server')
expiry_dt = None
if expiry:
try:
expiry_dt = datetime.datetime.utcfromtimestamp(expiry)
except (TypeError, ValueError):
authErr('Invalid expiry in returned token')
logging.debug(
'local_auth: got an access token for account "%s" that expires in %d sec',
account_id, expiry - time.mktime(now.timetuple()))
access_token = AccessToken(access_token, expiry_dt)
if _needs_refresh(access_token, now=now):
authErr('local_auth: the returned access token needs to be refreshed')
return access_token
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument(
'-v',
'--verbose',
action='store_true',
)
subparsers = parser.add_subparsers(dest='command')
put_parser = subparsers.add_parser('put')
put_parser.add_argument(
'-b',
'--bucket',
help=
('The bucket to schedule the build on. Typically the master name, e.g.'
' master.tryserver.chromium.linux.'),
required=True,
)
put_parser.add_argument(
'-c',
'--changes',
help='A flie to load a JSON list of changes dicts from.',
)
put_parser.add_argument(
'-n',
'--builder-name',
help='The builder to schedule the build on.',
required=True,
)
put_parser.add_argument(
'-p',
'--properties',
help='A file to load a JSON dict of properties from.',
)
args = parser.parse_args()
# TODO(smut): When more commands are implemented, refactor this.
assert args.command == 'put'
changes = []
if args.changes:
try:
with open(args.changes) as fp:
changes.extend(json.load(fp))
except (TypeError, ValueError):
sys.stderr.write('%s contained invalid JSON list.\n' %
args.changes)
raise
properties = {}
if args.properties:
try:
with open(args.properties) as fp:
properties.update(json.load(fp))
except (TypeError, ValueError):
sys.stderr.write('%s contained invalid JSON dict.\n' %
args.properties)
raise
authenticator = auth.get_authenticator_for_host(
BUILDBUCKET_URL,
auth.make_auth_config(use_oauth2=True),
)
http = authenticator.authorize(httplib2.Http())
http.force_exception_to_status_code = True
response, content = http.request(
PUT_BUILD_URL,
'PUT',
body=json.dumps({
'bucket':
args.bucket,
'parameters_json':
json.dumps({
'builder_name': args.builder_name,
'changes': changes,
'properties': properties,
}),
}),
headers={'Content-Type': 'application/json'},
)
if args.verbose:
print content
return response.status != 200
def monorail_get_auth_http(self):
auth_config = auth.extract_auth_config_from_options(self.options)
authenticator = auth.get_authenticator_for_host(
'bugs.chromium.org', auth_config)
return authenticator.authorize(httplib2.Http())
def __init__(self, deadline=20):
# Create a new API service for interacting with BigQuery
credentials = AppAssertionCredentials(scope=BigQueryClient.SCOPE)
http = credentials.authorize(httplib2.Http(timeout=deadline))
self.service = build('bigquery', 'v2', http=http)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument(
'-v',
'--verbose',
action='store_true',
)
subparsers = parser.add_subparsers(dest='command')
get_parser = subparsers.add_parser('get')
get_parser.add_argument(
'--id',
help='The ID of the build to get the status of.',
required=True,
)
put_parser = subparsers.add_parser('put')
put_parser.add_argument(
'-b',
'--bucket',
help=
('The bucket to schedule the build on. Typically the master name, e.g.'
' master.tryserver.chromium.linux.'),
required=True,
)
put_parser.add_argument(
'-c',
'--changes',
help='A flie to load a JSON list of changes dicts from.',
)
put_parser.add_argument(
'-n',
'--builder-name',
help='The builder to schedule the build on.',
required=True,
)
put_parser.add_argument(
'-p',
'--properties',
help=
('A file to load a JSON dict of properties from. Use "-" to pipe JSON '
'from another command.'),
)
args = parser.parse_args()
body = None
if args.command == 'get':
method = 'GET'
url = '%s/%s' % (BUILDBUCKET_API_URL, args.id)
elif args.command == 'put':
changes = []
if args.changes:
try:
with open(args.changes) as fp:
changes.extend(json.load(fp))
except (TypeError, ValueError):
sys.stderr.write('%s contained invalid JSON list.\n' %
args.changes)
raise
properties = {}
if args.properties:
try:
# Allow using pipes to stream properties from another command, e.g.
# echo '{"foo": "bar", "baz": 42}' | buildbucket.py -p -
if args.properties == '-':
properties.update(json.load(sys.stdin))
else:
with open(args.properties) as fp:
properties.update(json.load(fp))
except (TypeError, ValueError):
sys.stderr.write('%s contained invalid JSON dict.\n' %
args.properties)
raise
body = json.dumps({
'bucket':
args.bucket,
'parameters_json':
json.dumps({
'builder_name': args.builder_name,
'changes': changes,
'properties': properties,
}),
})
method = 'PUT'
url = BUILDBUCKET_API_URL
authenticator = auth.get_authenticator_for_host(
BUILDBUCKET_URL,
auth.make_auth_config(use_oauth2=True),
)
http = authenticator.authorize(httplib2.Http())
http.force_exception_to_status_code = True
response, content = http.request(
url,
method,
body=body,
headers={'Content-Type': 'application/json'},
)
if args.verbose:
print content
build_url = json.loads(content).get('build', {}).get('url')
if build_url:
print 'Build triggered on: %s' % build_url
return response.status != 200