def generate_signed_url(blob):
"""Generate signed URL for storage blob"""
credentials = default(
scopes=["https://www.googleapis.com/auth/cloud-platform"])[0]
signer = iam.Signer(
request=requests.Request(),
credentials=credentials,
service_account_email=os.getenv("FUNCTION_IDENTITY"),
)
# Create token-based service account credentials for signing
signing_credentials = service_account.IDTokenCredentials(
signer=signer,
token_uri="https://www.googleapis.com/oauth2/v4/token",
target_audience="",
service_account_email=os.getenv("FUNCTION_IDENTITY"),
)
# Cloud Functions service account must have Service Account Token Creator role
try:
url = blob.generate_signed_url(
version="v4",
expiration=datetime.timedelta(
hours=int(get_env('SIGNED_URL_EXPIRATION'))),
method="GET",
credentials=signing_credentials)
except exceptions.TransportError:
logging.error(
RuntimeError("Service account running the function must have IAM "
"roles/iam.serviceAccountTokenCreator."))
else:
print("Generated signed URL.")
return url
def delegated_credential(credentials, subject, scopes):
try:
admin_creds = credentials.with_subject(subject).with_scopes(scopes)
except AttributeError: # Looks like a compute creds object
# Refresh the boostrap credentials. This ensures that the information
# about this account, notably the email, is populated.
request = requests.Request()
credentials.refresh(request)
# Create an IAM signer using the bootstrap credentials.
signer = iam.Signer(request, credentials,
credentials.service_account_email)
# Create OAuth 2.0 Service Account credentials using the IAM-based
# signer and the bootstrap_credential's service account email.
admin_creds = service_account.Credentials(
signer,
credentials.service_account_email,
TOKEN_URI,
scopes=scopes,
subject=subject)
except Exception:
raise
return admin_creds
def _delegated_credential(
cls,
credentials: Credentials,
subject: str,
scopes: List[str],
) -> ServiceAccountCredentials:
try:
admin_credentials = credentials.with_subject(subject).with_scopes(
scopes)
except AttributeError:
# When inside GCP, the credentials provided by the metadata service are immutable.
# https://github.com/GoogleCloudPlatform/professional-services/tree/master/examples/gce-to-adminsdk
request = requests.Request()
credentials.refresh(request)
# Create an IAM signer using the bootstrap credentials.
signer = iam.Signer(
request,
credentials,
credentials.service_account_email,
)
# Create OAuth 2.0 Service Account credentials using the IAM-based
# signer and the bootstrap_credential's service account email.
admin_credentials = ServiceAccountCredentials(
signer,
credentials.service_account_email,
TOKEN_URI,
scopes=scopes,
subject=subject,
)
return admin_credentials
def _make_delegated_credentials(credentials, user_email, scopes):
"""Make delegated credentials.
Allows a service account to impersonate the user passed in `user_email`,
using a restricted set of scopes.
Args:
credentials (service_account.Credentials): The service account credentials.
user_email (str): The email for the user to impersonate.
scopes (list): A list of scopes.
Returns:
service_account.Credentials: The delegated credentials
"""
request = requests.Request()
credentials = with_scopes_if_required(credentials, _TOKEN_SCOPE)
credentials.refresh(request)
email = credentials.service_account_email
signer = iam.Signer(
request,
credentials,
email)
return service_account.Credentials(
signer,
email,
_TOKEN_URI,
scopes=scopes,
subject=user_email)
def test_key_id(self):
signer = iam.Signer(
mock.sentinel.request,
mock.sentinel.credentials,
mock.sentinel.service_account_email,
)
assert signer.key_id is None
def test_sign_bytes_failure(self):
request = make_request(http_client.UNAUTHORIZED)
credentials = make_credentials()
signer = iam.Signer(request, credentials, mock.sentinel.service_account_email)
with pytest.raises(exceptions.TransportError):
signer.sign("123")
def test_iam_signer(http_request, credentials):
credentials = credentials.with_scopes(
["https://www.googleapis.com/auth/iam"])
# Verify iamcredentials signer.
signer = iam.Signer(http_request, credentials,
credentials.service_account_email)
signed_blob = signer.sign("message")
assert isinstance(signed_blob, bytes)
def test_constructor(self):
request = mock.sentinel.request
credentials = mock.Mock(spec=google.auth.credentials.Credentials)
signer = iam.Signer(request, credentials,
mock.sentinel.service_account_email)
assert signer._request == mock.sentinel.request
assert signer._credentials == credentials
assert (signer._service_account_email ==
mock.sentinel.service_account_email)
def test_constructor(self):
request = mock.sentinel.request
credentials = mock.create_autospec(
google.auth.credentials.Credentials, instance=True
)
signer = iam.Signer(request, credentials, mock.sentinel.service_account_email)
assert signer._request == mock.sentinel.request
assert signer._credentials == credentials
assert signer._service_account_email == mock.sentinel.service_account_email
def test_sign_bytes(self):
signature = b"DEADBEEF"
encoded_signature = base64.b64encode(signature).decode("utf-8")
request = make_request(http_client.OK,
data={"signedBlob": encoded_signature})
credentials = make_credentials()
signer = iam.Signer(request, credentials,
mock.sentinel.service_account_email)
returned_signature = signer.sign("123")
assert returned_signature == signature
def test_sign_bytes(self):
signature = b"DEADBEEF"
encoded_signature = base64.b64encode(signature).decode("utf-8")
request = make_request(http_client.OK, data={"signedBlob": encoded_signature})
credentials = make_credentials()
signer = iam.Signer(request, credentials, mock.sentinel.service_account_email)
returned_signature = signer.sign("123")
assert returned_signature == signature
kwargs = request.call_args[1]
assert kwargs["headers"]["Content-Type"] == "application/json"
def test_sign_bytes(self):
signature = b'DEADBEEF'
encoded_signature = base64.b64encode(signature).decode('utf-8')
request = make_request(http_client.OK,
data={'signature': encoded_signature})
credentials = make_credentials()
signer = iam.Signer(request, credentials,
mock.sentinel.service_account_email)
returned_signature = signer.sign('123')
assert returned_signature == signature
def __init__(
self,
request,
target_audience,
token_uri=_DEFAULT_TOKEN_URI,
additional_claims=None,
service_account_email=None,
signer=None,
):
"""
Args:
request (google.auth.transport.Request): The object used to make
HTTP requests.
target_audience (str): The intended audience for these credentials,
used when requesting the ID Token. The ID Token's ``aud`` claim
will be set to this string.
token_uri (str): The OAuth 2.0 Token URI.
additional_claims (Mapping[str, str]): Any additional claims for
the JWT assertion used in the authorization grant.
service_account_email (str): Optional explicit service account to
use to sign JWT tokens.
By default, this is the default GCE service account.
signer (google.auth.crypt.Signer): The signer used to sign JWTs.
In case the signer is specified, the request argument will be
ignored.
"""
super(IDTokenCredentials, self).__init__()
if service_account_email is None:
sa_info = _metadata.get_service_account_info(request)
service_account_email = sa_info["email"]
self._service_account_email = service_account_email
if signer is None:
signer = iam.Signer(
request=request,
credentials=Credentials(),
service_account_email=service_account_email,
)
self._signer = signer
self._token_uri = token_uri
self._target_audience = target_audience
if additional_claims is not None:
self._additional_claims = additional_claims
else:
self._additional_claims = {}
def get_delegated_credentials(credentials, subject, scopes):
try:
request = requests.Request()
credentials.refresh(request)
signer = iam.Signer(request, credentials, config.GMAIL_SERVICE_ACCOUNT)
creds = service_account.Credentials(
signer=signer,
service_account_email=config.GMAIL_SERVICE_ACCOUNT,
token_uri=TOKEN_URI,
scopes=scopes,
subject=subject)
except Exception:
raise
return creds
def request_auth_token():
try:
credentials, project_id = google.auth.default(
scopes=['https://www.googleapis.com/auth/iam'])
request = gcp_requests.Request()
credentials.refresh(request)
signer = iam.Signer(request, credentials, config.DELEGATED_SA)
creds = service_account.Credentials(
signer=signer,
service_account_email=config.DELEGATED_SA,
token_uri=TOKEN_URI,
scopes=['https://www.googleapis.com/auth/cloud-platform'],
subject=config.DELEGATED_SA)
except Exception:
raise
return creds
def request_auth_token(self):
try:
credentials, project_id = google.auth.default(
scopes=["https://www.googleapis.com/auth/iam"])
request = gcp_requests.Request()
credentials.refresh(request)
signer = iam.Signer(request, credentials, config.DELEGATED_SA)
token_url = "https://accounts.google.com/o/oauth2/token" # nosec
creds = service_account.Credentials(
signer=signer,
service_account_email=config.DELEGATED_SA,
token_uri=token_url,
scopes=["https://www.googleapis.com/auth/cloud-platform"],
subject=config.DELEGATED_SA,
)
except Exception:
raise
return creds
def get_delegated_credential(delegated_account, scopes):
"""Build delegated credentials required for accessing the gsuite APIs.
Args:
delegated_account (str): The account to delegate the service account to
use.
scopes (list): The list of required scopes for the service account.
Returns:
service_account.Credentials: Credentials as built by
google.oauth2.service_account.
"""
request = requests.Request()
# Get the "bootstrap" credentials that will be used to talk to the IAM
# API to sign blobs.
bootstrap_credentials, _ = google.auth.default()
bootstrap_credentials = with_scopes_if_required(bootstrap_credentials,
list(CLOUD_SCOPES))
# Refresh the boostrap credentials. This ensures that the information about
# this account, notably the email, is populated.
bootstrap_credentials.refresh(request)
# Create an IAM signer using the bootstrap credentials.
signer = iam.Signer(request, bootstrap_credentials,
bootstrap_credentials.service_account_email)
# Create OAuth 2.0 Service Account credentials using the IAM-based signer
# and the bootstrap_credential's service account email.
delegated_credentials = service_account.Credentials(
signer,
bootstrap_credentials.service_account_email,
_TOKEN_URI,
scopes=scopes,
subject=delegated_account)
return delegated_credentials
def __update_credentials(credentials, subject, scopes):
try:
updated_credentials = credentials.with_subject(subject).with_scopes(
scopes)
except AttributeError:
# AttributeError implies that we are using GCE credentials so we cannot
# use with_subject and with_scopes. Following link has more details:
# https://github.com/GoogleCloudPlatform/professional-services/tree/master/examples/gce-to-adminsdk
request = requests.Request()
credentials.refresh(request)
signer = iam.Signer(request, credentials,
credentials.service_account_email)
updated_credentials = service_account.Credentials(
signer,
credentials.service_account_email,
GOOGLE_API['token_uri'],
scopes=scopes,
subject=subject)
except Exception:
raise
return updated_credentials
def delegated_credentials(credentials, scopes, subject=None):
"""
Generate scoped credentials. This needs the 'SA Token Creator' role for the SA
Source: https://stackoverflow.com/a/57092533
:param credentials: Credentials object to add scopes/subject to
:param scopes: Scopes to add to credentials
:param subject: Subject to add to credentials
:return: Updated credentials object with access token for scopes
"""
try:
# If using service account credentials from json file
updated_credentials = credentials.with_subject(subject).with_scopes(
scopes)
except AttributeError:
# If using GCE/GAE default credentials
request = requests.Request()
# Refresh the default credentials. This ensures that the information
# about this account, notably the email, is populated.
credentials.refresh(request)
# Create an IAM signer using the default credentials.
signer = iam.Signer(request, credentials,
credentials.service_account_email)
# Create OAuth 2.0 Service Account credentials using the IAM-based
# signer and the bootstrap_credential's service account email
updated_credentials = service_account.Credentials(
signer,
credentials.service_account_email,
TOKEN_URI,
scopes=scopes,
subject=subject)
return updated_credentials
def create_adminsdk_service():
credentials, project_id = google.auth.default(scopes=['https://www.googleapis.com/auth/iam'])
try:
request = requests.Request()
credentials.refresh(request)
signer = iam.Signer(request, credentials, config.SERVICE_ACCOUNT)
delegated_credentials = service_account.Credentials(
signer=signer,
service_account_email=config.SERVICE_ACCOUNT,
token_uri=TOKEN_URI,
scopes=config.SERVICE_ACCOUNT_SCOPES,
subject=config.USER_IMPERSONATION_EMAIL)
except Exception:
raise
gs_service = googleapiclient.discovery.build(
'admin', 'directory_v1', credentials=delegated_credentials,
cache_discovery=False)
rm_service = googleapiclient.discovery.build(
'cloudresourcemanager', 'v1beta1', credentials=delegated_credentials,
cache_discovery=False)
return gs_service, rm_service
def from_iam(cls, request, google_cred, service_account):
signer = iam.Signer(request, google_cred, service_account)
return _SigningProvider(signer, service_account)
def __init__(
self,
request,
target_audience,
token_uri=None,
additional_claims=None,
service_account_email=None,
signer=None,
use_metadata_identity_endpoint=False,
quota_project_id=None,
):
"""
Args:
request (google.auth.transport.Request): The object used to make
HTTP requests.
target_audience (str): The intended audience for these credentials,
used when requesting the ID Token. The ID Token's ``aud`` claim
will be set to this string.
token_uri (str): The OAuth 2.0 Token URI.
additional_claims (Mapping[str, str]): Any additional claims for
the JWT assertion used in the authorization grant.
service_account_email (str): Optional explicit service account to
use to sign JWT tokens.
By default, this is the default GCE service account.
signer (google.auth.crypt.Signer): The signer used to sign JWTs.
In case the signer is specified, the request argument will be
ignored.
use_metadata_identity_endpoint (bool): Whether to use GCE metadata
identity endpoint. For backward compatibility the default value
is False. If set to True, ``token_uri``, ``additional_claims``,
``service_account_email``, ``signer`` argument should not be set;
otherwise ValueError will be raised.
quota_project_id (Optional[str]): The project ID used for quota and
billing.
Raises:
ValueError:
If ``use_metadata_identity_endpoint`` is set to True, and one of
``token_uri``, ``additional_claims``, ``service_account_email``,
``signer`` arguments is set.
"""
super(IDTokenCredentials, self).__init__()
self._quota_project_id = quota_project_id
self._use_metadata_identity_endpoint = use_metadata_identity_endpoint
self._target_audience = target_audience
if use_metadata_identity_endpoint:
if token_uri or additional_claims or service_account_email or signer:
raise ValueError(
"If use_metadata_identity_endpoint is set, token_uri, "
"additional_claims, service_account_email, signer arguments"
" must not be set"
)
self._token_uri = None
self._additional_claims = None
self._signer = None
if service_account_email is None:
sa_info = _metadata.get_service_account_info(request)
self._service_account_email = sa_info["email"]
else:
self._service_account_email = service_account_email
if not use_metadata_identity_endpoint:
if signer is None:
signer = iam.Signer(
request=request,
credentials=Credentials(),
service_account_email=self._service_account_email,
)
self._signer = signer
self._token_uri = token_uri or _DEFAULT_TOKEN_URI
if additional_claims is not None:
self._additional_claims = additional_claims
else:
self._additional_claims = {}
if _whitelist:
_whitelist = [p.strip() for p in _whitelist.split(',')]
_username = os.getenv('AUTH_USERNAME')
_password = os.getenv('AUTH_PASSWORD')
# For service accounts using the Compute Engine metadata service, which is the
# case for Cloud Function service accounts, service_account_email isn't
# available until refresh is called.
_adc_credentials.refresh(GRequest())
# Since the Compute Engine metadata service doesn't expose the service
# account key, we use the IAM signBlob API to sign instead. In order for this
# to work, the Cloud Function's service account needs the "Service Account
# Actor" role.
_signer = iam.Signer(
GRequest(), _adc_credentials, _adc_credentials.service_account_email)
def requires_auth(f):
"""Decorator to enforce Basic authentication on requests."""
@wraps(f)
def decorated(*args, **kwargs):
auth = request.authorization
if _is_auth_enabled():
if not auth or not _check_auth(auth.username, auth.password):
return ('Could not verify your access level for that URL.\n'
'You have to login with proper credentials.', 401,
{'WWW-Authenticate': 'Basic realm="Login Required"'})
return f(*args, **kwargs)
return decorated
def main():
"""Sends an email with a Google Cloud Storage signed URL of BQ Query results.
Creates BQ table, runs a SQL query, and exports the results to Cloud Storage as a CSV.
Generates signing credentials for the CSV and sends an email with a link to the signed URL.
Args:
None
Returns:
None
"""
#Create BQ and Storage Client
bq_client = bigquery.Client(credentials=credentials())
storage_client = storage.Client(credentials=credentials())
# Set variables
timestr = time.strftime("%Y%m%d%I%M%S")
project = "report-scheduling"
dataset_id = "bq_exports"
file_name = "daily_export_" + timestr
csv_name = file_name + ".csv"
table_id = "report-scheduling.bq_exports." + file_name
bucket_name = "bq_email_exports"
from_email = "*****@*****.**"
to_emails = "*****@*****.**"
# Create a BQ table
schema = [
bigquery.SchemaField("url", "STRING", mode="REQUIRED"),
bigquery.SchemaField("view_count", "INTEGER", mode="REQUIRED"),
]
# Make an API request to create table
table = bq_client.create_table(bigquery.Table(table_id, schema=schema))
print("Created table {}.{}.{}".format(table.project, table.dataset_id,
table.table_id))
# Run query on that table
job_config = bigquery.QueryJobConfig(destination=table_id)
# Define the SQL query
sql = """
SELECT
CONCAT(
'https://stackoverflow.com/questions/',
CAST(id as STRING)) as url,
view_count
FROM `bigquery-public-data.stackoverflow.posts_questions`
WHERE tags like '%google-bigquery%'
ORDER BY view_count DESC
LIMIT 10
"""
# Start the query, passing in the extra configuration
query_job = bq_client.query(sql, job_config=job_config)
# Wait for the job to complete
query_job.result()
print("Query results loaded to the table {}".format(table_id))
# Export table data as CSV to GCS
destination_uri = "gs://{}/{}".format(bucket_name, csv_name)
dataset_ref = bq_client.dataset(dataset_id, project=project)
table_ref = dataset_ref.table(file_name)
extract_job = bq_client.extract_table(
table_ref,
destination_uri,
# Location must match that of the source table
location="US",
)
# Waits for job to complete
extract_job.result()
print("Exported {}:{}.{} to {}".format(project, dataset_id, table_id,
destination_uri))
# Generate a v4 signed URL for downloading a blob
bucket = storage_client.bucket(bucket_name)
blob = bucket.blob(csv_name)
signing_credentials = None
# If running on GCF, generate signing credentials
# Service account running the GCF must have Service Account Token Creator role
if os.getenv("IS_LOCAL") is None:
signer = iam.Signer(
request=requests.Request(),
credentials=credentials(),
service_account_email=os.getenv("FUNCTION_IDENTITY"),
)
# Create Token-based service account credentials for signing
signing_credentials = service_account.IDTokenCredentials(
signer=signer,
token_uri="https://www.googleapis.com/oauth2/v4/token",
target_audience="",
service_account_email=os.getenv("FUNCTION_IDENTITY"),
)
url = blob.generate_signed_url(
version="v4",
# This URL is valid for 24 hours, until the next email
expiration=datetime.timedelta(hours=24),
# Allow GET requests using this URL
method="GET",
# Signing credentials; if None falls back to json credentials in local environment
credentials=signing_credentials,
)
print("Generated GET signed URL.")
# Create email message through SendGrid with link to signed URL
message = Mail(
from_email=from_email,
to_emails=to_emails,
subject="Daily BQ export",
html_content="<p> Your daily BigQuery export from Google Cloud Platform \
is linked <a href={}>here</a>.</p>".format(url),
)
# Send email
try:
sg = SendGridAPIClient(os.environ.get('SENDGRID_API_KEY'))
response = sg.send(message)
print(response.status_code)
except Exception as e:
print(e.message)