def complete_multipart_upload():
"""
Complete multipart upload
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
missing = {"key", "uploadId", "parts"}.difference(set(params))
if missing:
raise UserError("missing required arguments: {}".format(list(missing)))
default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration(
params.get("expires_in"),
max_limit=default_expires_in,
default=default_expires_in,
)
try:
BlankIndex.complete_multipart_upload(
params["key"], params["uploadId"], params["parts"], expires_in=expires_in
),
except InternalError as e:
return flask.jsonify({"message": e.message}), e.code
return flask.jsonify({"message": "OK"}), 200
def init_multipart_upload():
"""
Initialize a multipart upload request
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
blank_index = BlankIndex(file_name=params["file_name"])
default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration(
params.get("expires_in"),
max_limit=default_expires_in,
default=default_expires_in,
)
response = {
"guid": blank_index.guid,
"uploadId": BlankIndex.init_multipart_upload(
blank_index.guid + "/" + params["file_name"], expires_in=expires_in
),
}
return flask.jsonify(response), 201
def test_make_signed_url_missing_configuration_key(app, indexd_client):
"""
Test BlankIndex make_signed_url with a missing configuration key
"""
uploader = MagicMock()
current_app = flask.current_app
expected_value = copy.deepcopy(current_app.config)
del expected_value["AZ_BLOB_CONTAINER_URL"]
del expected_value["DATA_UPLOAD_BUCKET"]
indexed_file_location = indexd_client["indexed_file_location"]
with patch.object(current_app, "config", expected_value):
assert current_app.config == expected_value
blank_index = BlankIndex(uploader=uploader)
assert blank_index
with patch(
"fence.blueprints.data.indexd.AzureBlobStorageIndexedFileLocation.get_signed_url"
):
with patch(
"fence.blueprints.data.indexd.S3IndexedFileLocation.get_signed_url"
):
with pytest.raises(InternalError):
signed_url = blank_index.make_signed_url(
file_name="some file name",
protocol=indexed_file_location)
def test_complete_multipart_upload_missing_key(app, indexd_client):
"""
Test BlankIndex complete_multipart_upload with a missing configuration key
"""
uploader = MagicMock()
current_app = flask.current_app
expected_value = copy.deepcopy(current_app.config)
del expected_value["DATA_UPLOAD_BUCKET"]
with patch.object(current_app, "config", expected_value):
assert current_app.config == expected_value
blank_index = BlankIndex(uploader=uploader)
assert blank_index
with pytest.raises(InternalError):
blank_index.complete_multipart_upload(
key="some key",
uploadId="some id",
parts=[
{
"Etag": "1234567",
"PartNumber": 1
},
{
"Etag": "4321234",
"PartNumber": 2
},
],
)
def upload_data_file():
"""
Return a presigned URL for use with uploading a data file.
See the documentation on the entire flow here for more info:
https://github.com/uc-cdis/cdis-wiki/tree/master/dev/gen3/data_upload
"""
# make new record in indexd, with just the `uploader` field (and a GUID)
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
blank_index = BlankIndex(file_name=params["file_name"])
expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
response = {
"guid":
blank_index.guid,
"url":
blank_index.make_signed_url(params["file_name"],
expires_in=expires_in),
}
return flask.jsonify(response), 201
def test_init_multipart_upload(app, indexd_client):
"""
Test BlankIndex init_multipart_upload
"""
uploader = MagicMock()
blank_index = BlankIndex(uploader=uploader)
assert blank_index
with patch(
"fence.blueprints.data.indexd.S3IndexedFileLocation.init_multipart_upload"
):
blank_index.init_multipart_upload(key="some key")
def test_generate_aws_presigned_url_for_part(app, indexd_client):
"""
Test BlankIndex generate_aws_presigned_url_for_part
"""
uploader = MagicMock()
blank_index = BlankIndex(uploader=uploader)
assert blank_index
with patch(
"fence.blueprints.data.indexd.S3IndexedFileLocation.generate_presigned_url_for_part_upload"
):
blank_index.generate_aws_presigned_url_for_part(key="some key",
uploadId="some id",
partNumber=1,
expires_in=10)
def test_init_multipart_upload_missing_configuration_key(app, indexd_client):
"""
test BlankIndex init_multipart_upload with a missing configuration key
"""
uploader = MagicMock()
current_app = flask.current_app
expected_value = copy.deepcopy(current_app.config)
del expected_value["DATA_UPLOAD_BUCKET"]
with patch.object(current_app, "config", expected_value):
assert current_app.config == expected_value
blank_index = BlankIndex(uploader=uploader)
assert blank_index
with pytest.raises(InternalError):
blank_index.init_multipart_upload(key="some key")
def generate_multipart_upload_presigned_url():
"""
Generate multipart upload presigned url
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
missing = {"key", "uploadId", "partNumber"}.difference(set(params))
if missing:
raise UserError("missing required arguments: {}".format(list(missing)))
default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration(
params.get("expires_in"),
max_limit=default_expires_in,
default=default_expires_in,
)
response = {
"presigned_url": BlankIndex.generate_aws_presigned_url_for_part(
params["key"],
params["uploadId"],
params["partNumber"],
expires_in=expires_in,
)
}
return flask.jsonify(response), 200
def test_make_signed_url(app, indexd_client):
"""
Test BlankIndex make_signed_url with a missing configuration key
"""
uploader = MagicMock()
indexed_file_location = indexd_client["indexed_file_location"]
blank_index = BlankIndex(uploader=uploader)
assert blank_index
with patch(
"fence.blueprints.data.indexd.AzureBlobStorageIndexedFileLocation.get_signed_url"
):
with patch(
"fence.blueprints.data.indexd.S3IndexedFileLocation.get_signed_url"
):
signed_url = blank_index.make_signed_url(
file_name="some file name", protocol=indexed_file_location)
def test_generate_aws_presigned_url_for_part_missing_key(app, indexd_client):
"""
Test BlankIndex generate_aws_presigned_url_for_part with a missing configuration key
"""
uploader = MagicMock()
current_app = flask.current_app
expected_value = copy.deepcopy(current_app.config)
del expected_value["DATA_UPLOAD_BUCKET"]
with patch.object(current_app, "config", expected_value):
assert current_app.config == expected_value
blank_index = BlankIndex(uploader=uploader)
assert blank_index
with pytest.raises(InternalError):
blank_index.generate_aws_presigned_url_for_part(key="some key",
uploadId="some id",
partNumber=1,
expires_in=10)
def test_blank_index_set_uploader(app, indexd_client):
"""
Test BlankIndex with an uploader
"""
uploader = MagicMock()
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
blank_index = BlankIndex(uploader=uploader)
assert blank_index
def test_complete_multipart_upload(app, indexd_client):
"""
Test BlankIndex complete_multipart_upload
"""
uploader = MagicMock()
blank_index = BlankIndex(uploader=uploader)
assert blank_index
with patch(
"fence.blueprints.data.indexd.S3IndexedFileLocation.complete_multipart_upload"
):
blank_index.complete_multipart_upload(
key="some key",
uploadId="some id",
parts=[
{
"Etag": "1234567",
"PartNumber": 1
},
{
"Etag": "4321234",
"PartNumber": 2
},
],
)
def upload_data_file():
"""
Return a presigned URL for use with uploading a data file.
See the documentation on the entire flow here for more info:
https://github.com/uc-cdis/cdis-wiki/tree/master/dev/gen3/data_upload
"""
# make new record in indexd, with just the `uploader` field (and a GUID)
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
authorized = False
authz_err_msg = "Auth error when attempting to get a presigned URL for upload. User must have '{}' access on '{}'."
authz = params.get("authz")
uploader = None
if authz:
# if requesting an authz field, using new authorization method which doesn't
# rely on uploader field, so clear it out
uploader = ""
authorized = flask.current_app.arborist.auth_request(
jwt=get_jwt(),
service="fence",
methods=["create", "write-storage"],
resources=authz,
)
if not authorized:
logger.error(authz_err_msg.format("create' and 'write-storage", authz))
else:
# no 'authz' was provided, so fall back on 'file_upload' logic
authorized = flask.current_app.arborist.auth_request(
jwt=get_jwt(),
service="fence",
methods=["file_upload"],
resources=["/data_file"],
)
if not authorized:
logger.error(authz_err_msg.format("file_upload", "/data_file"))
if not authorized:
raise Forbidden(
"You do not have access to upload data. You either need "
"general file uploader permissions or create and write-storage permissions "
"on the authz resources you specified (if you specified any)."
)
blank_index = BlankIndex(
file_name=params["file_name"], authz=params.get("authz"), uploader=uploader
)
default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration(
params.get("expires_in"),
max_limit=default_expires_in,
default=default_expires_in,
)
response = {
"guid": blank_index.guid,
"url": blank_index.make_signed_url(params["file_name"], expires_in=expires_in),
}
return flask.jsonify(response), 201
