def test_fetch_id_token_invalid_cred_file(mock_init, monkeypatch):
not_json_file = os.path.join(os.path.dirname(__file__),
"../data/public_cert.pem")
monkeypatch.setenv(environment_vars.CREDENTIALS, not_json_file)
with pytest.raises(exceptions.DefaultCredentialsError):
request = mock.Mock()
id_token.fetch_id_token(request, "https://pubsub.googleapis.com")
def _request(self, url, method, **kwargs):
"""Makes a request to an application protected by Identity-Aware Proxy.
For valid values for method and kwargs, see
https://requests.readthedocs.io/en/master/api/?highlight=request#requests.request.
Args:
url: The Identity-Aware Proxy-protected URL to fetch.
method: The request method to use.
**kwargs: Any of the parameters defined for the requests.request.
Returns:
The response as a request.Response object.
Raises:
Same exceptions as request.request.
"""
# Obtain an OpenID Connect (OIDC) token from metadata server or
# using service account.
google_open_id_connect_token = id_token.fetch_id_token(
google.auth.transport.requests.Request(), self.client_id)
# Fetch the Identity-Aware Proxy-protected URL, including an
# Authorization header containing "Bearer " followed by a
# Google-issued OpenID Connect token for the service account.
headers = kwargs.setdefault('headers', {})
headers['Authorization'] = f'Bearer {google_open_id_connect_token}'
return requests.request(method, url, **kwargs)
def obtain_id_token(self):
from google.auth.transport.requests import Request
from google.oauth2 import id_token
from google.auth.exceptions import DefaultCredentialsError
client_id = os.environ.get(IAP_CLIENT_ID, None)
jwt_token = None
if not client_id:
self.log.info(
"No IAP_CLIENT_ID provided, skipping custom IAP authentication"
)
return jwt_token
try:
self.log.debug("Obtaining JWT token for %s." + client_id)
jwt_token = id_token.fetch_id_token(Request(), client_id)
self.log.info("Obtained JWT token for MLFLOW connectivity.")
except DefaultCredentialsError as ex:
self.log.warning(
str(ex) +
(" Note that this authentication method does not work with default"
" credentials obtained via 'gcloud auth application-default login'"
" command. Refer to documentation on how to configure service account"
" locally"
" (https://cloud.google.com/docs/authentication/production#manually)"
))
except Exception as e:
self.log.error("Failed to obtain IAP access token. " + str(e))
finally:
return jwt_token
def make_iap_request(url, client_id, method='GET', **kwargs):
"""Makes a request to an application protected by Identity-Aware Proxy.
Args:
url: The Identity-Aware Proxy-protected URL to fetch.
client_id: The client ID used by Identity-Aware Proxy.
method: The request method to use
('GET', 'OPTIONS', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE')
**kwargs: Any of the parameters defined for the request function:
https://github.com/requests/requests/blob/master/requests/api.py
If no timeout is provided, it is set to 90 by default.
Returns:
The page body, or raises an exception if the page couldn't be retrieved.
"""
# Set the default timeout, if missing
if 'timeout' not in kwargs:
kwargs['timeout'] = 90
# Obtain an OpenID Connect (OIDC) token from metadata server or using service
# account
google_open_id_connect_token = id_token.fetch_id_token(Request(), client_id)
# Fetch the Identity-Aware Proxy-protected URL, including an
# Authorization header containing "Bearer " followed by a
# Google-issued OpenID Connect token for the service account.
resp = requests.request(method, url,headers={'Authorization': 'Bearer {}'.format(google_open_id_connect_token)}, **kwargs)
if resp.status_code == 403:
raise Exception('Service account does not have permission to access the IAP-protected application.')
elif resp.status_code != 200:
raise Exception('Bad response from application: {!r} / {!r} / {!r}'.format(resp.status_code, resp.headers, resp.text))
else:
return resp.text
def _request(client_id, url, method, **kwargs):
"""Makes a request to an application protected by Identity-Aware Proxy.
Args:
client_id: The client ID used by Identity-Aware Proxy.
url: The Identity-Aware Proxy-protected URL to fetch.
method: The request method to use
('GET', 'OPTIONS', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE')
**kwargs: Any of the parameters defined for the request function:
https://github.com/requests/requests/blob/master/requests/api.py
Returns:
The Response.
"""
# Obtain an OpenID Connect (OIDC) token from metadata server or
# using service account.
google_open_id_connect_token = id_token.fetch_id_token(
Request(), client_id)
# Fetch the Identity-Aware Proxy-protected URL, including an
# Authorization header containing "Bearer " followed by a
# Google-issued OpenID Connect token for the service account.
headers = kwargs.setdefault('headers', {})
headers['Authorization'] = 'Bearer {}'.format(google_open_id_connect_token)
response = requests.request(method, url, **kwargs)
return response
def _refresh_token(self):
""" Refreshes Google ID token and persists it in memory """
# Use static token if available
if self._static_token:
self._token = self._static_token
return
from google.oauth2.id_token import fetch_id_token
try:
self._token = fetch_id_token(self._request, audience="feast.dev")
return
except DefaultCredentialsError:
pass
# Try to use Google Auth library to find ID Token
from google import auth as google_auth
try:
credentials, _ = google_auth.default(["openid", "email"])
credentials.refresh(self._request)
if hasattr(credentials, "id_token"):
self._token = credentials.id_token
return
except DefaultCredentialsError:
pass # Could not determine credentials, skip
# Raise exception otherwise
raise RuntimeError(
"Could not determine Google ID token. Ensure that a service account can be found by setting"
" the GOOGLE_APPLICATION_CREDENTIALS environmental variable to its path."
)
def hello_world():
try:
token = id_token.fetch_id_token(http_request,
"https://pubsub.googleapis.com")
except Exception as e:
return str(e)
# return credentials.token
return token
def hello():
"""Return a friendly HTTP greeting."""
try:
token = id_token.fetch_id_token(http_request,
"https://pubsub.googleapis.com")
except Exception as e:
return str(e)
#return credentials.token
return token
def list_planning_items(): # noqa: E501
service_account_token = id_token.fetch_id_token(
Request(), config.PLANNING_ENGINE_FUNCTION_URL)
function_url = config.PLANNING_ENGINE_FUNCTION_URL
# Provide the token in the request to the receiving function
function_headers = {'Authorization': f'bearer {service_account_token}'}
res = requests.get(function_url, headers=function_headers)
if 200 <= res.status_code < 300:
res = json.loads(res.content)
workitems_query = db_client.query(kind='WorkItem')
workitems = list(workitems_query.fetch())
engineer_query = db_client.query(kind='Engineer')
engineers = list(engineer_query.fetch())
engineers_by_id = {
engineer.key.id_or_name: engineer
for engineer in engineers
}
workitems_by_id = {
workitem.key.id_or_name: workitem
for workitem in workitems
}
planning = []
for planning_item in res['result']:
planning.append(
create_planning_item(engineers_by_id, workitems_by_id,
planning_item))
unplanned_engineers = []
for engineer_item in res['unplanned_engineers']:
unplanned_engineers.append(
create_engineer(engineers_by_id[engineer_item]))
unplanned_workitems = []
for work_item in res['unplanned_workitems']:
unplanned_workitems.append(
create_workitem(workitems_by_id[work_item]))
response = PlanningItemsList(
items=planning,
unplanned_engineers=unplanned_engineers,
unplanned_workitems=unplanned_workitems,
links=HALSelfRef(f'{request.url_root}plannings'))
return make_response(jsonify(response), 200,
{'Cache-Control': 'private, max-age=300'})
else:
response = Error(
500, 'Er is een fout opgetreden bij het genereren van de planning')
return make_response(response, 500)
def create_token(audience) -> str:
"""
Create Google OpenID token with the given audience
:param audience: url endpoints for which this token can be successfully authenticated
:return:
"""
request = google.auth.transport.requests.Request()
# https://github.com/googleapis/google-auth-library-python/blob/ca8d98ab2e5277e53ab8df78beb1e75cdf5321e3/google/oauth2/id_token.py#L168-L252
return id_token.fetch_id_token(request, audience)
def make_post_iap_request(self, url, client_id, json, **kwargs):
"""Makes a POST request to an application protected by Identity-Aware Proxy.
Args:
url: The Identity-Aware Proxy-protected URL to fetch.
client_id: The client ID used by Identity-Aware Proxy.
json: A JSON payload containing any additional data to be included with the POST request.
**kwargs: Any of the parameters defined for the request function:
https://github.com/requests/requests/blob/master/requests/api.py
If no timeout is provided, it is set to 90 by default.
Returns:
The page body, or raises an exception if the page couldn't be retrieved.
"""
self.logger.log(logging.DEBUG, "Entered make_post_iap_request method")
self.logger.log(
logging.DEBUG,
f"make_post_iap_request method params. url: {url}, client_id: {client_id}, json: {json}"
)
# Set the default timeout, if missing
if 'timeout' not in kwargs:
kwargs['timeout'] = 90
# Obtain an OpenID Connect (OIDC) token from metadata server or using service
# account.
# try to get the id token from the auth_service
google_open_id_connect_token = auth_service.get_id_token(
Request(), client_id)
if google_open_id_connect_token is None:
google_open_id_connect_token = id_token.fetch_id_token(
Request(), client_id)
# Fetch the Identity-Aware Proxy-protected URL, including an
# Authorization header containing "Bearer " followed by a
# Google-issued OpenID Connect token for the service account.
resp = requests.request(
'POST',
url,
headers={
'Authorization':
'Bearer {}'.format(google_open_id_connect_token),
'Content-Type': 'application/json'
},
json=json,
**kwargs)
if resp.status_code == 403:
raise Exception('Service account does not have permission to '
'access the IAP-protected application.')
elif resp.status_code != 200:
raise Exception(
'Bad response from application: {!r} / {!r} / {!r}'.format(
resp.status_code, resp.headers, resp.text))
else:
return resp.text
def make_request():
url = request.form['url']
token = id_token.fetch_id_token(reqs.Request(), url)
resp = requests.get(url,
headers={'Authorization': 'Bearer {}'.format(token)})
message = 'Response when calling {}:\n\n'.format(url)
message += resp.text
return message, 200, {'Content-type': 'text/plain'}
def query(url_query, audience, method='GET', body=None):
open_id_connect_token = id_token.fetch_id_token(Request(),
audience=audience)
resp = requests.request(
method,
url_query,
headers={'Authorization': 'Bearer {}'.format(open_id_connect_token)},
json=body)
return resp.text
def handle_request(proxied_request):
"""Proxy the given request to the URL in the Forward-Host header with an
Authorization header set using an OIDC bearer token for the Cloud
Function's service account. If the header is not present, return a 400
error.
"""
host = proxied_request.headers.get(HOST_HEADER)
if not host:
return 'Required header {} not present'.format(HOST_HEADER), 400
scheme = proxied_request.headers.get('X-Forwarded-Proto', 'https')
url = '{}://{}{}'.format(scheme, host, proxied_request.path)
headers = dict(proxied_request.headers)
# Check path against whitelist.
path = proxied_request.path
if not path:
path = '/'
# TODO: Implement proper wildcarding for paths.
if '*' not in _whitelist and path not in _whitelist:
logging.warn('Rejected {} {}, not in whitelist'.format(
proxied_request.method, url))
return 'Requested path {} not in whitelist'.format(path), 403
#global _oidc_token
#if not _oidc_token or _oidc_token.is_expired():
# _oidc_token = _get_google_oidc_token()
# logging.info('Renewed OIDC bearer token for {}'.format(
# _adc_credentials.service_account_email))
client_id = os.getenv('CLIENT_ID')
_oidc_token = id_token.fetch_id_token(GRequest(), client_id)
# Add the Authorization header with the OIDC token.
headers['Authorization'] = 'Bearer {}'.format(_oidc_token)
# We don't want to forward the Host header.
headers.pop('Host', None)
request = Request(proxied_request.method,
url,
headers=headers,
data=proxied_request.data)
# Send the proxied request.
prepped = request.prepare()
logging.info('{} {}'.format(prepped.method, prepped.url))
resp = _session.send(prepped)
# Strip hop-by-hop headers and Content-Encoding.
headers = _strip_hop_by_hop_headers(resp.headers)
headers.pop('Content-Encoding', None)
return resp.content, resp.status_code, headers.items()
def test_fetch_id_token_from_explicit_cred_json_file(mock_init, monkeypatch):
monkeypatch.setenv(environment_vars.CREDENTIALS, SERVICE_ACCOUNT_FILE)
def mock_refresh(self, request):
self.token = "id_token"
with mock.patch.object(google.oauth2.service_account.IDTokenCredentials,
"refresh", mock_refresh):
request = mock.Mock()
token = id_token.fetch_id_token(request,
"https://pubsub.googleapis.com")
assert token == "id_token"
def test_fetch_id_token(monkeypatch):
mock_cred = mock.MagicMock()
mock_cred.token = "token"
mock_req = mock.Mock()
with mock.patch(
"google.oauth2.id_token.fetch_id_token_credentials", return_value=mock_cred
) as mock_fetch:
token = id_token.fetch_id_token(mock_req, ID_TOKEN_AUDIENCE)
mock_fetch.assert_called_once_with(ID_TOKEN_AUDIENCE, request=mock_req)
mock_cred.refresh.assert_called_once_with(mock_req)
assert token == "token"
def test_fetch_id_token_from_metadata_server():
def mock_init(self, request, audience, use_metadata_identity_endpoint):
assert use_metadata_identity_endpoint
self.token = "id_token"
with mock.patch.multiple(
google.auth.compute_engine.IDTokenCredentials,
__init__=mock_init,
refresh=mock.Mock(),
):
request = mock.Mock()
token = id_token.fetch_id_token(request,
"https://pubsub.googleapis.com")
assert token == "id_token"
def make_get_iap_request(self, url, client_id, **kwargs):
"""Makes a GET request to an application protected by Identity-Aware Proxy.
Args:
url: The Identity-Aware Proxy-protected URL to fetch.
client_id: The client ID used by Identity-Aware Proxy.
**kwargs: Any of the parameters defined for the request function:
https://github.com/requests/requests/blob/master/requests/api.py
If no timeout is provided, it is set to 90 by default.
Returns:
The page body, or raises an exception if the page couldn't be retrieved.
"""
self.logger.log(logging.DEBUG, "Entered make_get_iap_request method")
self.logger.log(
logging.DEBUG,
f"make_get_iap_request method params. url: {url}, client_id: {client_id}"
)
# Set the default timeout, if missing
if 'timeout' not in kwargs:
kwargs['timeout'] = 90
# try to get the id token from the auth_service
google_open_id_connect_token = auth_service.get_id_token(
Request(), client_id)
if google_open_id_connect_token is None:
google_open_id_connect_token = id_token.fetch_id_token(
Request(), client_id)
resp = requests.request(
'GET',
url,
headers={
'Authorization':
'Bearer {}'.format(google_open_id_connect_token)
},
**kwargs)
if resp.status_code == 403:
raise Exception('Service account does not have permission to '
'access the IAP-protected application.')
elif resp.status_code != 200:
raise Exception(
'Bad response from application: {!r} / {!r} / {!r}'.format(
resp.status_code, resp.headers, resp.text))
else:
return resp.text
def query(url_query, audience, method='GET', body=None):
open_id_connect_token = id_token.fetch_id_token(Request(),
audience=audience)
resp = requests.request(
method,
url_query,
headers={'Authorization': 'Bearer {}'.format(open_id_connect_token)},
json=body)
try:
resp.json()
except:
app.logger.error("Error while querying : {} - {}".format(
url_query, resp.reason))
pass
return resp
def fetch_test_token(target_audience):
"""
Fetch test token from a given service account
verify with:
os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
return: (success, token)
"""
try:
os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
except KeyError:
# path not yet set
set_google_cred()
try:
request = requests.Request()
iden_token = id_token.fetch_id_token(request, target_audience)
return True, iden_token
except:
return False, None
def call(self, data=None):
from google.oauth2.id_token import fetch_id_token
auth_req = GCRequest()
id_token = fetch_id_token(auth_req, 'https://' + self.service_url)
headers = {"Authorization": f"Bearer {id_token}"}
if data:
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
data = json.dumps(data)
request = requests.post('https://' + self.service_url,
data=data,
headers=headers)
logger.info(f'Cloud function {self.service_url}')
logger.info(request.content.decode('utf-8'))
res = request.json()
return res
def make_iap_request(url, client_id, method='GET', **kwargs):
if 'timeout' not in kwargs:
kwargs['timeout'] = 90
google_open_id_connect_token = id_token.fetch_id_token(
Request(), client_id)
print(google_open_id_connect_token)
resp = requests.request(
method,
url,
headers={
'Authorization': 'Bearer {}'.format(google_open_id_connect_token)
},
**kwargs)
if resp.status_code == 403:
raise Exception(
'Service account does not have permission to access the IAP-protected application.'
)
elif resp.status_code != 200:
raise Exception(
'Bad response from application: {!r} / {!r} / {!r}'.format(
resp.status_code, resp.headers, resp.text))
else:
return resp.text
def _make_iap_request(url, method='GET', **kwargs):
logging.warning(f'Will ge token')
google_open_id_connect_token = id_token.fetch_id_token(
Request(), client_id)
logging.warning(f'Will call request')
resp = requests.request(
method,
url,
headers={
'Authorization': 'Bearer {}'.format(google_open_id_connect_token)
},
**kwargs)
logging.warning(f'It get response')
if resp.status_code == 403:
raise Exception(
'Service account does not have permission to access the IAP-protected application.'
)
elif resp.status_code != 200:
raise Exception(
'Bad response from application: {!r} / {!r} / {!r}'.format(
resp.status_code, resp.headers, resp.text))
else:
return resp.text
def auth_token(self) -> str:
return id_token.fetch_id_token(Request(), self.config.client_id)
def _get_token():
client_id = _get_client_id(tracking_uri)
open_id_connect_token = id_token.fetch_id_token(Request(), client_id)
return open_id_connect_token
def test_fetch_id_token_no_cred_json_file(mock_init, monkeypatch):
monkeypatch.delenv(environment_vars.CREDENTIALS, raising=False)
with pytest.raises(exceptions.DefaultCredentialsError):
request = mock.Mock()
id_token.fetch_id_token(request, "https://pubsub.googleapis.com")
def get_identity_token(audience):
request = google.auth.transport.requests.Request()
token = id_token.fetch_id_token(request, audience)
return token
def auth_token(self) -> str:
if self.token_valid():
return self.jwt_token
return id_token.fetch_id_token(self.google_request,
self.config.client_id)
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from google.oauth2 import id_token
import google.auth.transport.requests
import requests
import sys
import urllib
from urllib.parse import urlparse
host = urlparse(sys.argv[1]).netloc.split(':')[0]
response = requests.get("https://%s/" % ( host, ), allow_redirects = False)
query = urllib.parse.urlparse(response.headers["location"]).query
client_id = urllib.parse.parse_qs(query)["client_id"][0]
request = google.auth.transport.requests.Request()
open_id_connect_token = id_token.fetch_id_token(request, client_id)
print(open_id_connect_token)
def function_handler(event, context):
file = event
bucket_name = file['bucket']
file_name = file['name']
process_data = 'gs://' + bucket_name + '/' + file_name
print('The file is about to be processed: ' + str(process_data))
data = json.dumps({"conf": {"gs_location": process_data}})
print('The airflow payload: ' + str(data))
# Authenticate with Google Cloud.
# See: https://cloud.google.com/docs/authentication/getting-started
credentials, _ = google.auth.default(
scopes=['https://www.googleapis.com/auth/cloud-platform'])
authed_session = google.auth.transport.requests.AuthorizedSession(
credentials)
project_id = 'helical-decoder-322615'
location = 'us-east1'
composer_environment = 'airflow'
dag_name = 'dataproc_job_flow_dag'
environment_url = (
'https://composer.googleapis.com/v1beta1/projects/{}/locations/{}'
'/environments/{}').format(project_id, location, composer_environment)
composer_response = authed_session.request('GET', environment_url)
environment_data = composer_response.json()
airflow_uri = environment_data['config']['airflowUri']
print(airflow_uri)
# The Composer environment response does not include the IAP client ID.
# Make a second, unauthenticated HTTP request to the web server to get the
# redirect URI.
redirect_response = requests.get(airflow_uri, allow_redirects=False)
redirect_location = redirect_response.headers['location']
print(redirect_response.headers)
# Extract the client_id query parameter from the redirect.
parsed = six.moves.urllib.parse.urlparse(redirect_location)
query_string = six.moves.urllib.parse.parse_qs(parsed.query)
client_id = query_string['client_id'][0]
webserver_url = airflow_uri + '/api/experimental/dags/' + dag_name + '/dag_runs'
# Obtain an OpenID Connect (OIDC) token from metadata server or using service account.
google_open_id_connect_token = id_token.fetch_id_token(
google.auth.transport.requests.Request(), client_id)
print('google_open_id_connect_token:', google_open_id_connect_token)
# Fetch the Identity-Aware Proxy-protected URL, including an
# Authorization header containing "Bearer " followed by a
# Google-issued OpenID Connect token for the service account.
returncode = subprocess.run([
"curl", "-X", "POST", webserver_url, "-H",
"Authorization: Bearer {}".format(google_open_id_connect_token),
"--insecure", "-d", data
])
print('returncode:', returncode)
if returncode == 0:
return {'statusCode': 200, 'body': json.dumps('Function is working!')}
