def get(http_request, path, root=METADATA_ROOT, recursive=None):
"""Fetch a resource from the metadata server.
Args:
path: A string indicating the resource to retrieve. For example,
'instance/service-accounts/defualt'
http_request: A callable that matches the method
signature of httplib2.Http.request. Used to make the request to the
metadataserver.
root: A string indicating the full path to the metadata server root.
recursive: A boolean indicating whether to do a recursive query of
metadata. See
https://cloud.google.com/compute/docs/metadata#aggcontents
Returns:
A dictionary if the metadata server returns JSON, otherwise a string.
Raises:
httplib2.Httplib2Error if an error corrured while retrieving metadata.
"""
url = urlparse.urljoin(root, path)
url = util._add_query_parameter(url, 'recursive', recursive)
response, content = http_request(url, headers=METADATA_HEADERS)
if response.status == http_client.OK:
decoded = _helpers._from_bytes(content)
if response['content-type'] == 'application/json':
return json.loads(decoded)
else:
return decoded
else:
raise httplib2.HttpLib2Error(
'Failed to retrieve {0} from the Google Compute Engine'
'metadata service. Response:\n{1}'.format(url, response))
def FetchBuilderJsonFromMilo(master,
builder,
limit=100,
service_account_file=None): # pragma: no cover
LOGGER.debug('Fetching buildbot json for %s/%s from milo', master, builder)
body = {'master': master, 'builder': builder, 'limit': limit}
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json',
}
http = httplib2.Http(timeout=300)
if service_account_file:
creds = infra_libs.get_signed_jwt_assertion_credentials(
service_account_file, scope=OAUTH_SCOPES)
creds.authorize(http)
resp, content = http.request(MILO_JSON_ENDPOINT,
method='POST',
headers=headers,
body=json.dumps(body))
if resp.status != 200:
raise httplib2.HttpLib2Error('Invalid response status: %s\n%s' %
(resp.status, content))
# Strip off jsonp header.
data = json.loads(content[4:])
builds = [
json.loads(base64.b64decode(build['data'])) for build in data['builds']
]
return {build['number']: build for build in builds}
def GetRefs(self, refs_regex=None, filter_regex=None):
"""Get a dict of refs from the given git repo, like ls-remote.
Args:
refs_regex: regex list for which refs to monitor.
filter_regex: List of regex substitutions for matching filter refs (if
any) to corresponding monitored refs (used to filter unwanted commits
from monitoring).
Returns:
A dict of "ref:commit" and a dict of "ref:filter_ref" mappings.
"""
refs = {}
filters = {}
refs_url = '%s/+refs?format=TEXT' % self._api_url
self.logger.debug('Refs request: %s', refs_url)
try:
resp, content = self.http.request(refs_url, headers=self.headers)
if resp.status >= 400:
raise httplib2.HttpLib2Error('Invalid response status %d' %
resp.status)
except httplib2.HttpLib2Error as e:
self.logger.exception('Failed refs request: %s (%s). Response: %r',
refs_url, e, resp)
return refs, filters
splitter = re.compile(r'(?P<commit>[0-9a-fA-F]+)\s+(?P<ref>[^\s]+)$')
ref_res = [(re.compile('.*'), None)]
if refs_regex:
ref_res = [[re.compile(ref_reg + '$'), None]
for ref_reg in refs_regex]
for idx, filter_reg in enumerate(filter_regex or []):
ref_res[idx][1] = filter_reg
all_refs = []
for line in content.splitlines():
m = splitter.match(line)
if m:
ref = m.group('ref')
all_refs.append(ref)
for ref_re in ref_res:
if ref_re[0].match(ref):
refs[ref] = m.group('commit')
# Try to calculate the corresponding filter ref (if any) based on
# the name of this monitored ref and the defined substitution, e.g.
# refs/heads/* => refs/upstream/heads/*).
if ref_re[1]:
filters[ref] = re.sub(ref_re[0], ref_re[1], ref)
else:
self.logger.debug('Unable to split line:\n%s', line)
# Remove any bogus filter refs, otherwise using them to specify a commit
# range will generate an error and not find any commits.
for key, val in filters.items():
if not val in all_refs:
self.logger.debug('Filter ref "%s" not found.' % val)
filters[key] = None
return refs, filters
def testGetCredentialsFromEnvLocal(self):
self.mox.StubOutWithMock(gce, 'AppAssertionCredentials')
credentials = self.mox.CreateMockAnything()
gce.AppAssertionCredentials(connection.SCOPE).AndReturn(credentials)
credentials.authorize(mox.IsA(httplib2.Http))
credentials.refresh(mox.IsA(httplib2.Http)).AndRaise(
httplib2.HttpLib2Error())
self.mox.ReplayAll()
self.assertIs(None, helper.get_credentials_from_env())
self.mox.VerifyAll()
def testGetDatastoreFromEnvNone(self):
self.mox.StubOutWithMock(httplib2, 'Http')
http = self.mox.CreateMockAnything()
httplib2.Http().AndReturn(http)
http.request('http://metadata/computeMetadata/v1/project/project-id',
headers={
'X-Google-Metadata-Request': 'True'
}).AndRaise(httplib2.HttpLib2Error())
self.mox.ReplayAll()
self.assertEquals(None, helper.get_dataset_from_env())
self.mox.VerifyAll()
def test_get_ip_info_vmi_conn_error(self):
self.ipinfo.nova_client.servers.list(search_opts=mox.IgnoreArg()
).AndReturn(test_data.instances)
url = urljoin(self.ipinfo.contrail_url, self.ipinfo.contrail_vmi_path)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request(url, 'GET').AndRaise(httplib2.HttpLib2Error())
self.m.ReplayAll()
instances = self.ipinfo.get_ip_info()
self.assertEqual(2, len(instances))
for instance in instances:
self.assertFalse(hasattr(instance, 'fixed_ips'))
self.assertFalse(hasattr(instance, 'floating_ips'))
self.m.VerifyAll()
self.m.UnsetStubs()
def GetCommitDiff(self, commit):
"""Get the text diff introduced by the argument commit."""
diff_url = '%s/+/%s%%5E%%21/?format=TEXT' % (self._api_url, commit)
self.logger.debug('Diff request: %s', diff_url)
try:
resp, content = self.http.request(diff_url, headers=self.headers)
if resp.status >= 400:
raise httplib2.HttpLib2Error('Invalid response status %d' %
resp.status)
return b64decode(content)
except httplib2.HttpLib2Error as e:
self.logger.exception('Failed Diff request %s: %s', diff_url,
str(e))
return None
def GetLogDict(log_range, limit):
url = '%s/+log/%s?format=JSON&n=%d' % (self._api_url, log_range,
limit)
self.logger.debug('Log request: %s', url)
try:
resp, content = self.http.request(url, headers=self.headers)
if resp.status >= 400:
raise httplib2.HttpLib2Error('Invalid response status %d' %
resp.status)
# Skip the first 5 bytes of the response due to JSON anti-XSSI prefix.
json_txt = content[5:]
return json.loads(json_txt)
except httplib2.HttpLib2Error as e:
self.logger.exception('Failed log request %s: %s', url, str(e))
return None
def crop_image(url, width=0, height=0, img_type='jpg', quality=100, **kwargs):
"""
:param url: source image url
:param width: cropped width
:param height: cropped height
:param img_type: cropped image type
:param quality: cropped image quality
"""
face_detect = kwargs.get('face_detect')
if not os.path.exists(_data_path):
os.mkdir(_data_path)
h = httplib2.Http('.cache')
try:
(resp, content) = h.request(url, 'GET')
if content[0:20].find('The requested URL') != -1:
raise httplib2.HttpLib2Error('URL cannot find in server')
except httplib2.HttpLib2Error, err:
raise Exception('%s request error: %s' % (url, str(err)))
def _FetchFromBuildbucketImpl(project,
bucket_name,
builder,
service_account_file=None): # pragma: no cover
request_pb = rpc_pb2.SearchBuildsRequest()
request_pb.predicate.builder.project = project
request_pb.predicate.builder.bucket = bucket_name
request_pb.predicate.builder.builder = builder
request_pb.predicate.status = common_pb2.ENDED_MASK
request_pb.fields.paths.extend([
'builds.*.number',
'builds.*.status',
'builds.*.input.gitiles_commit.id',
])
headers = {
'Accept': 'application/prpc; encoding=binary',
'Content-Type': 'application/prpc; encoding=binary',
}
http = httplib2.Http(timeout=300)
creds = None
if service_account_file:
creds = infra_libs.get_signed_jwt_assertion_credentials(
service_account_file, scope=OAUTH_SCOPES)
elif luci_auth.available():
creds = luci_auth.LUCICredentials(scopes=OAUTH_SCOPES)
if creds:
creds.authorize(http)
resp, content = http.request(_BUILDBUCKET_SEARCH_ENDPOINT_V2.format(
buildbucket_instance=_DEFAULT_BUILDBUCKET_INSTANCE),
method='POST',
headers=headers,
body=request_pb.SerializeToString())
grpc_code = resp.get('X-Prpc-Grpc-Code'.lower())
if grpc_code != '0':
raise httplib2.HttpLib2Error('Invalid GRPC exit code: %s\n%s' %
(grpc_code, content))
response_pb = rpc_pb2.SearchBuildsResponse()
response_pb.ParseFromString(content)
return response_pb
def GetMergedChanges(self,
since=None,
limit=None,
start=None,
fields=None):
"""Query gerrit for merged changes (i.e. commits).
API docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
https://gerrit-review.googlesource.com/Documentation/user-search.html
Args:
since: Only get commits after this datetime() (UTC assumed).
limit: Maximum number of results to return.
start: Offset in the result set to start at.
fields: A list of additional output fields (cause more database lookups
and slows down query response time).
"""
query = '%s/changes/?q=status:merged' % self._api_url
if since:
query = '%s+since:{%s}' % (
query, urllib.quote('%s' % self.GenerateTimeStamp(since)))
if start:
query = '%s&S=%d' % (query, start)
if limit:
query = '%s&n=%d' % (query, limit)
if fields:
query = '%s&%s' % (query, '&'.join(['o=%s' % p for p in fields]))
self.logger.debug('Gerrit commits request: %s', query)
try:
resp, content = self.http.request(query, headers=self.headers)
if resp.status >= 400:
raise httplib2.HttpLib2Error('Invalid response status %d' %
resp.status)
# Skip the first 5 bytes of the response due to JSON anti-XSSI prefix.
json_txt = content[5:]
except httplib2.HttpLib2Error as e:
self.logger.exception('Failed gerrit request %s: %s', query,
str(e))
return {}
return json.loads(json_txt)
def request(self, api_url, query={}, credentials=None, app_access=False, method="GET"):
if credentials: self.credentials = credentials
if not app_access and self.credentials.access_token_expired:
raise httplib2.HttpLib2Error('Access token expired')
h = httplib2.Http()
if app_access:
request_url = api_url + "?access_token=" + self.access_token
else:
request_url = api_url + "?access_token=" + self.credentials.access_token
for item in query.iteritems():
request_url += '&' + str(item[0])+ '=' + str(item[1])
head, content = h.request(request_url, method=method)
if head.get('status') == '200':
return (head, content)
else:
logger.error("AccessTokenCredentialsError: %s", content)
raise AccessTokenCredentialsError(content)
def test_retries_transient_errors(self):
self.mock_httplib2([
({
'uri': 'http://localhost/123'
}, httplib2.HttpLib2Error()),
({
'uri': 'http://localhost/123'
}, (httplib2.Response({
'status': 408,
'reason': 'client timeout',
}), 'client timeout')),
({
'uri': 'http://localhost/123'
}, (httplib2.Response({
'status': 500,
'reason': 'server error',
}), 'server error')),
({
'uri': 'http://localhost/123'
}, (httplib2.Response({}), 'response body')),
])
response = net.request('http://localhost/123', max_attempts=4)
self.assertEqual('response body', response)
def GetCommitDetails(self, ref):
"""Get a the details of the given commit ref.
Args:
ref: The commit ref to get the details for.
Returns:
Dict of gitiles commit details:
commit, tree, parents, author, committer, message, tree_diff
"""
commit_url = '%s/+/%s?format=JSON' % (self._api_url, ref)
self.logger.debug('Commit request: %s', commit_url)
try:
resp, content = self.http.request(commit_url, headers=self.headers)
if resp.status >= 400:
raise httplib2.HttpLib2Error('Invalid response status %d' %
resp.status)
# Skip the first 5 bytes of the response due to JSON anti-XSSI prefix.
json_txt = content[5:]
except httplib2.HttpLib2Error as e:
self.logger.exception('Failed Commit request %s: %s', commit_url,
str(e))
return {}
return json.loads(json_txt)
cookies = response.get('set-cookie')
jsessionid = re.findall(r'(JSESSIONID=[0-9A-F]+);', cookies)
if len(jsessionid) > 0:
self._jsession = jsessionid[0]
if response is not None and response.get('status') == '404':
r, _ = self._http.request(self._server)
if self._server.rstrip('/') != r.get('content-location',
self._server).rstrip('/'):
old_server = self._server
self._server = r.get('content-location').rstrip('/')
return self._exec(uri.replace(old_server, ''), method, body)
else:
raise httplib2.HttpLib2Error(
'%s %s %s' % (uri, response.status, response.reason))
if is_xnat_error(content):
catch_error(content)
return content
def _get_json(self, uri):
""" Specific Interface._exec method to retrieve data.
It forces the data format to csv and then puts it back to a
json-like format.
Parameters
----------
uri: string
URI of the resource to be accessed. e.g. /REST/projects
def test_fail_exception(self, _sleep): self.http._http.request.side_effect = httplib2.HttpLib2Error() self.assertRaises(httplib2.HttpLib2Error, self.http.request, 'http://foo/') self.http._http.request.assert_has_calls([ self._MOCK_REQUEST ] * 5)
def _exec(self, uri, method='GET', body=None, headers=None):
""" A wrapper around a simple httplib2.request call that:
- avoids repeating the server url in the request
- deals with custom caching mechanisms
- manages a user session with cookies
- catches and broadcast specific XNAT errors
Parameters
----------
uri: string
URI of the resource to be accessed. e.g. /REST/projects
method: GET | PUT | POST | DELETE
HTTP method.
body: string
HTTP message body
headers: dict
Additional headers for the HTTP request.
"""
if headers is None:
headers = {}
self._get_entry_point()
uri = join_uri(self._server, uri)
if DEBUG:
print(uri)
# using session authentication
headers['cookie'] = self._jsession
headers['connection'] = 'keep-alive'
# reset the memcache when client changes something on the server
if method in ['PUT', 'DELETE']:
self._memcache = {}
# Initialize these to default values.
response = None
info = None
content = None
if self._mode == 'online' and method == 'GET':
if time.time() - self._memcache.get(uri, 0) < self._memtimeout:
if DEBUG:
print('send: GET CACHE %s' % uri)
info, content = self._http.cache.get(uri).split('\r\n\r\n', 1)
self._memcache[uri] = time.time()
else:
response, content = self._http.request(uri, method, body,
headers)
self._memcache[uri] = time.time()
elif self._mode == 'offline' and method == 'GET':
cached_value = self._http.cache.get(uri)
if cached_value is not None:
if DEBUG:
print('send: GET CACHE %s' % uri)
info, content = cached_value.split('\r\n\r\n', 1)
else:
try:
self._http.timeout = 10
response, content = self._http.request(
uri, method, body, headers)
self._http.timeout = None
self._memcache[uri] = time.time()
except Exception as e:
catch_error(e)
else:
response, content = self._http.request(uri, method, body, headers)
if DEBUG:
if response is None:
response = httplib2.Response(email.message_from_string(info))
print('reply: %s %s from cache') % (response.status,
response.reason)
for key in response.keys():
print('header: %s: %s') % (key.title(), response.get(key))
if response is not None and 'set-cookie' in response:
cookies = response.get('set-cookie')
jsessionid = re.findall(r'(JSESSIONID=[0-9A-F]+);', cookies)
if len(jsessionid) > 0:
self._jsession = jsessionid[0]
if response is not None and response.get('status') == '404':
r, _ = self._http.request(self._server)
if self._server.rstrip('/') != r.get('content-location',
self._server).rstrip('/'):
old_server = self._server
self._server = r.get('content-location').rstrip('/')
return self._exec(uri.replace(old_server, ''), method, body)
else:
raise httplib2.HttpLib2Error(
'%s %s %s' % (uri, response.status, response.reason))
if is_xnat_error(content):
print(response.keys())
print(response.get("status"))
catch_error(content)
return content
def _raise_http_err(master, builder, **kwargs):
if builder == 'builder1':
return {'build1': 'success'}
raise httplib2.HttpLib2Error()
def testPollHTTPError(self):
self.SetNextException(httplib2.HttpLib2Error('bad stuff'))
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
tailer.Poll() # no error raised
def testPollHTTPError_IsLast(self):
self.SetNextException(httplib2.HttpLib2Error('bad stuff'))
tailer = cloudbuild_logs.LogTailer('bucket', 'log-build-id.txt')
with self.assertRaisesRegex(api_exceptions.CommunicationError,
'bad stuff'):
tailer.Poll(is_last=True) # the last poll will raise the error
