def get_signed_url(
self,
protocol,
action,
expires_in,
force_signed_url=True,
r_pays_project=None,
file_name=None,
):
if self.index_document.get("authz"):
action_to_permission = {
"upload": "write-storage",
"download": "read-storage",
}
if not self.check_authz(action_to_permission[action]):
raise Unauthorized(
f"Either you weren't logged in or you don't have "
f"{action_to_permission[action]} permission "
f"on {self.index_document['authz']} for fence")
else:
if self.public_acl and action == "upload":
raise Unauthorized(
"Cannot upload on public files while using acl field")
# don't check the authorization if the file is public
# (downloading public files with no auth is fine)
if not self.public_acl and not self.check_authorization(action):
raise Unauthorized(
f"You don't have access permission on this file: {self.file_id}"
)
if action is not None and action not in SUPPORTED_ACTIONS:
raise NotSupported("action {} is not supported".format(action))
return self._get_signed_url(protocol, action, expires_in,
force_signed_url, r_pays_project,
file_name)
def get_signed_url(self, protocol, action, expires_in):
if not self.public and not self.check_authorization(action):
raise Unauthorized("You don't have access permission on this file")
if action is not None and action not in SUPPORTED_ACTIONS:
raise NotSupported("action {} is not supported".format(action))
return self._get_signed_url(protocol, action, expires_in)
def from_url(url):
protocol = urlparse(url).scheme
if (protocol is not None) and (protocol not in SUPPORTED_PROTOCOLS):
raise NotSupported(
"The specified protocol {} is not supported".format(protocol))
if protocol == "s3":
return S3IndexedFileLocation(url)
elif protocol == "gs":
return GoogleStorageIndexedFileLocation(url)
return IndexedFileLocation(url)
def get_prefix_for_google_proxy_groups():
"""
Return a string prefix for Google proxy groups based on configuration.
Returns:
str: prefix for proxy groups
"""
prefix = config.get("GOOGLE_GROUP_PREFIX")
if not prefix:
raise NotSupported(
"GOOGLE_GROUP_PREFIX must be set in the configuration. "
"This namespaces the Google groups for security and safety.")
return prefix
def return_link(action, urls):
protocol = flask.request.args.get('protocol', None)
expires = min(int(flask.request.args.get('expires_in', 3600)), 3600)
if (protocol is not None) and (protocol not in SUPPORTED_PROTOCOLS):
raise NotSupported("The specified protocol is not supported")
if len(urls) == 0:
raise NotFound("Can't find any location for the data")
for url in urls:
location = urlparse(url)
if check_protocol(protocol, location.scheme):
return resolve_url(url, location, expires, action)
raise NotFound(
"Can't find a location for the data with given request arguments.")
def get_google_project_valid_users_and_service_accounts(
project_id, google_cloud_manager, membership=None
):
"""
Gets google project members of type
USER or SERVICE_ACCOUNT and raises an error if it finds a member
that isn't one of those types.
Args:
project_id (str): Google project ID
google_cloud_manager(GoogleCloudManager): cloud manager instance
membership (List(GooglePolicyMember): pre-calculated list of members,
Will make call to Google API if membership is None
Return:
List[cirrus.google_cloud.iam.GooglePolicyMember]: Members on the
google project
Raises:
NotSupported: Member is invalid type
"""
try:
members = membership or google_cloud_manager.get_project_membership(project_id)
for member in members:
if not (
member.member_type == GooglePolicyMember.SERVICE_ACCOUNT
or member.member_type == GooglePolicyMember.USER
):
raise NotSupported(
"Member {} has invalid type: {}".format(
member.email_id, member.member_type
)
)
users = [
member
for member in members
if member.member_type == GooglePolicyMember.USER
]
service_accounts = [
member
for member in members
if member.member_type == GooglePolicyMember.SERVICE_ACCOUNT
]
return users, service_accounts
except Exception as exc:
logger.error(
"validity of Google Project (id: {}) members "
"could not complete. Details: {}".format(project_id, exc)
)
raise
def get_signed_url(self,
protocol,
action,
expires_in,
force_signed_url=True):
if self.public and action == "upload":
raise Unauthorized("Cannot upload on public files")
# don't check the authorization if the file is public
# (downloading public files with no auth is fine)
if not self.public and not self.check_authorization(action):
raise Unauthorized("You don't have access permission on this file")
if action is not None and action not in SUPPORTED_ACTIONS:
raise NotSupported("action {} is not supported".format(action))
return self._get_signed_url(protocol, action, expires_in,
force_signed_url)
def resolve_url(url, location, expires, action):
protocol = location.scheme
if protocol == 's3':
aws_creds = current_app.config['AWS_CREDENTIALS']
if 'AWS_CREDENTIALS' in current_app.config and len(aws_creds) > 0:
buckets = flask.current_app.config['S3_BUCKETS']
if location.netloc not in buckets.keys():
raise Unauthorized('permission denied for bucket')
if buckets[location.netloc] not in aws_creds:
raise Unauthorized('permission denied for bucket')
credential_key = buckets[location.netloc]
url = current_app.boto.presigned_url(
location.netloc,
location.path.strip('/'),
expires,
aws_creds[credential_key],
ACTION_DICT[protocol][action],
)
elif protocol not in ['http', 'https']:
raise NotSupported('protocol {} in URL {} is not supported'.format(
protocol, url))
return flask.jsonify(dict(url=url))
def wrapper(self, provider, *args, **kwargs):
if provider not in self.clients:
raise NotSupported("This backend is not supported by the system!")
return f(self, provider, *args, **kwargs)
def give_service_account_billing_access_if_necessary(
sa_private_key, r_pays_project=None, default_billing_project=None
):
"""
Give the Service Account (whose key is provided) the privilege to bill to the
given project. If a project is not provided and there is a configured Google project
to bill to, we will use that.
Args:
sa_private_key (dict): JSON key in Google Credentials File format:
.. code-block:: JavaScript
{
"type": "service_account",
"project_id": "project-id",
"private_key_id": "some_number",
"private_key": "-----BEGIN PRIVATE KEY-----\n....
=\n-----END PRIVATE KEY-----\n",
"client_email": "<api-name>[email protected]",
"client_id": "...",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/...<api-name>api%40project-id.iam.gserviceaccount.com"
}
r_pays_project (str, optional): The Google Project identifier to bill to
default_billing_project (str, optional): the default The Google Project
identifier to bill to if r_pays_project is None
"""
if not r_pays_project and not default_billing_project:
sa_account_id = sa_private_key.get("client_email")
raise UserError(
"You did NOT provide a `userProject` for requester pays billing, "
"so we could not create a custom role in that project to provide "
"the necessary service account ({}) billing permission. "
"Our main service account ({}) will need valid permissions in the "
"project you supplied to create a custom role and change the project "
"IAM policy. There is no configured default billing project so you must "
"provide a `userProject` query parameter.".format(
sa_account_id, config["CIRRUS_CFG"].get("GOOGLE_ADMIN_EMAIL")
)
)
# use configured project if it exists and no user project was given
is_default_billing = False
if default_billing_project and not r_pays_project:
r_pays_project = default_billing_project
is_default_billing = True
if r_pays_project:
sa_account_id = sa_private_key.get("client_email")
try:
# attempt to create custom role that gives
# the SA access to bill the project provided
# NOTE: this may fail if our fence SA doesn't have the right permissions
# to add this role and update the project policy
with GoogleCloudManager(project_id=r_pays_project) as g_cloud_manager:
g_cloud_manager.give_service_account_billing_access(
sa_account_id, project_id=r_pays_project
)
except Exception as exc:
logger.error(
"Unable to create a custom role in Google Project {} to "
"give Google service account {} rights to bill the project. Error: {}".format(
r_pays_project, sa_account_id, exc
)
)
if is_default_billing:
raise InternalError(
"Fence has a configured Google Project for requester pays billing ({}), "
"but could not create a custom role in that project to provide "
"the necessary service account ({}) billing permission. It could be that "
"the Fence admin service account ({}) does not have valid permissions in the "
"project.".format(
r_pays_project,
sa_account_id,
config["CIRRUS_CFG"].get("GOOGLE_ADMIN_EMAIL"),
)
)
else:
raise NotSupported(
"You provided {} as a `userProject` for requester pays billing, "
"but we could not create a custom role in that project to provide "
"the necessary service account ({}) billing permission. It could be that "
"our main service account ({}) does not have valid permissions in the "
"project you supplied to create a custom role and change the project IAM policy.".format(
r_pays_project,
sa_account_id,
config["CIRRUS_CFG"].get("GOOGLE_ADMIN_EMAIL"),
)
)
logger.info(
"Created a custom role in Google Project {} to "
"give Google service account {} rights to bill the project.".format(
r_pays_project, sa_account_id
)
)
def create_keypairs(provider):
"""
Generate a keypair for user
:query expires_in: expiration time in seconds, default and max is 30 days
**Example:**
.. code-block:: http
POST /credentials/cdis/?expires_in=3600 HTTP/1.1
Content-Type: application/json
Accept: application/json
cdis:
.. code-block:: JavaScript
{
"key_id": result,
"api_key": result
}
google:
(JSON key in Google Credentials File format)
.. code-block:: JavaScript
{
"type": "service_account",
"project_id": "project-id",
"private_key_id": "some_number",
"private_key": "-----BEGIN PRIVATE KEY-----\n....
=\n-----END PRIVATE KEY-----\n",
"client_email": "<api-name>[email protected]",
"client_id": "...",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/...<api-name>api%40project-id.iam.gserviceaccount.com"
}
other:
.. code-block:: JavaScript
{
"access_key": "8DGW9LyC0D4nByoWo6pp",
"secret_key": "1lnkGScEH8Vr4EC6QnoqLK1PqRWPNqIBJkH6Vpgx"
}
"""
client_id = getattr(flask.g, 'client_id', None)
if provider == 'cdis':
# requestor is user if client_id is not set
if client_id is None:
client_id = str(flask.g.user.id)
# fence identifies access_token endpoint, openid is the default
# scope for service endpoints
default_scope = ['fence', 'openid']
content_type = flask.request.headers.get('Content-Type')
if content_type == 'application/x-www-form-urlencoded':
scope = flask.request.form.getlist('scope')
else:
try:
scope = (
json.loads(flask.request.data)
.get('scope')
) or []
except ValueError:
scope = []
if not isinstance(scope, list):
scope = scope.split(',')
scope.extend(default_scope)
for s in scope:
if s not in USER_ALLOWED_SCOPES:
raise NotSupported('Scope {} is not supported'.format(s))
expires_in = min(
int(flask.request.args.get('expires_in', 2592000)),
2592000
)
api_key, claims = create_api_key(
flask.g.user, flask.current_app.keypairs[0], expires_in, scope,
client_id
)
return flask.jsonify(dict(key_id=claims['jti'], api_key=api_key))
elif provider == 'google':
with GoogleCloudManager() as g_cloud:
key = _get_google_access_key(g_cloud)
return flask.jsonify(key)
else:
return flask.jsonify(flask.current_app.storage_manager.create_keypair(
provider, flask.g.user
))