def complete_multipart_upload():
"""
Complete multipart upload
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
missing = {"key", "uploadId", "parts"}.difference(set(params))
if missing:
raise UserError("missing required arguments: {}".format(list(missing)))
expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
try:
BlankIndex.complete_multipart_upload(params["key"],
params["uploadId"],
params["parts"],
expires_in=expires_in),
except InternalError as e:
return flask.jsonify({"message": e.message}), e.code
return flask.jsonify({"message": "OK"}), 200
def generate_multipart_upload_presigned_url():
"""
Generate multipart upload presigned url
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
missing = {"key", "uploadId", "partNumber"}.difference(set(params))
if missing:
raise UserError("missing required arguments: {}".format(list(missing)))
expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
response = {
"presigned_url":
BlankIndex.generate_aws_presigned_url_for_part(
params["key"],
params["uploadId"],
params["partNumber"],
expires_in=expires_in,
)
}
return flask.jsonify(response), 200
def upload_data_file():
"""
Return a presigned URL for use with uploading a data file.
See the documentation on the entire flow here for more info:
https://github.com/uc-cdis/cdis-wiki/tree/master/dev/gen3/data_upload
"""
# make new record in indexd, with just the `uploader` field (and a GUID)
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
blank_index = BlankIndex(file_name=params["file_name"])
expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
response = {
"guid":
blank_index.guid,
"url":
blank_index.make_signed_url(params["file_name"],
expires_in=expires_in),
}
return flask.jsonify(response), 201
def init_multipart_upload():
"""
Initialize a multipart upload request
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
blank_index = BlankIndex(file_name=params["file_name"])
expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
response = {
"guid": blank_index.guid,
"uploadId": BlankIndex.init_multipart_upload(
blank_index.guid + "/" + params["file_name"], expires_in=expires_in
),
}
return flask.jsonify(response), 201
def upload_data_file():
"""
Return a presigned URL for use with uploading a data file.
See the documentation on the entire flow here for more info:
https://github.com/uc-cdis/cdis-wiki/tree/master/dev/gen3/data_upload
"""
# make new record in indexd, with just the `uploader` field (and a GUID)
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
authorized = False
authz_err_msg = "Auth error when attempting to get a presigned URL for upload. User must have '{}' access on '{}'."
authz = params.get("authz")
uploader = None
if authz:
# if requesting an authz field, using new authorization method which doesn't
# rely on uploader field, so clear it out
uploader = ""
authorized = flask.current_app.arborist.auth_request(
jwt=get_jwt(),
service="fence",
methods=["create", "write-storage"],
resources=authz,
)
if not authorized:
logger.error(
authz_err_msg.format("create' and 'write-storage", authz))
else:
# no 'authz' was provided, so fall back on 'file_upload' logic
authorized = flask.current_app.arborist.auth_request(
jwt=get_jwt(),
service="fence",
methods=["file_upload"],
resources=["/data_file"],
)
if not authorized:
logger.error(authz_err_msg.format("file_upload", "/data_file"))
if not authorized:
raise Forbidden(
"You do not have access to upload data. You either need "
"general file uploader permissions or create & write-storage permissions "
"on the authz resources you specified (if you specified any).")
blank_index = BlankIndex(file_name=params["file_name"],
authz=params.get("authz"),
uploader=uploader)
expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
response = {
"guid":
blank_index.guid,
"url":
blank_index.make_signed_url(params["file_name"],
expires_in=expires_in),
}
return flask.jsonify(response), 201
def force_update_google_link(DB, username, google_email, expires_in=None):
"""
WARNING: This function circumvents Google Auth flow, and should only be
used for internal testing!
WARNING: This function assumes that a user already has a proxy group!
Adds user's google account to proxy group and/or updates expiration for
that google account's access.
WARNING: This assumes that provided arguments represent valid information.
This BLINDLY adds without verification. Do verification
before this.
Specifically, this ASSUMES that the proxy group provided belongs to the
given user and that the user has ALREADY authenticated to prove the
provided google_email is also their's.
Args:
DB
username (str): Username to link with
google_email (str): Google email to link to
Raises:
NotFound: Linked Google account not found
Unauthorized: Couldn't determine user
Returns:
Expiration time of the newly updated google account's access
"""
cirrus_config.update(**config["CIRRUS_CFG"])
db = SQLAlchemyDriver(DB)
with db.session as session:
user_account = query_for_user(session=session, username=username)
if user_account:
user_id = user_account.id
proxy_group_id = user_account.google_proxy_group_id
else:
raise Unauthorized(
"Could not determine authed user "
"from session. Unable to link Google account."
)
user_google_account = (
session.query(UserGoogleAccount)
.filter(UserGoogleAccount.email == google_email)
.first()
)
if not user_google_account:
user_google_account = add_new_user_google_account(
user_id, google_email, session
)
# timestamp at which the SA will lose bucket access
# by default: use configured time or 7 days
expiration = int(time.time()) + config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
if expires_in:
is_valid_expiration(expires_in)
# convert it to timestamp
requested_expiration = int(time.time()) + expires_in
expiration = min(expiration, requested_expiration)
force_update_user_google_account_expiration(
user_google_account, proxy_group_id, google_email, expiration, session
)
session.commit()
return expiration
