def test_delete_call_not_successful(app, public_bucket_indexd_client):
"""
Test fence.blueprints.data.indexd.IndexedFile delete_files fails
"""
class MockResponse:
"""
Mock Response from requests lib
"""
def __init__(self, data, status_code=200):
"""
Setup Mock Response
"""
self.data = data
self.status_code = status_code
def json(self):
"""
Mock json() call
"""
return self.data
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.resources.user.user_session.UserSession.create_initial_token"
):
with patch(
"fence.blueprints.data.indexd.requests.delete",
return_value=MockResponse(data=None, status_code=503),
):
indexed_file = IndexedFile(file_id="some id")
assert indexed_file.delete()
def test_internal_get_signed_url_no_location_match(
app, supported_action, supported_protocol,
indexd_client_accepting_record):
"""
Test fence.blueprints.data.indexd.IndexedFile call to _get_signed_url with location not found
"""
indexd_record_with_non_public_authz_and_public_acl_populated = {
"urls": [f"{supported_protocol}://some/location"],
"authz": ["/programs/DEV/projects/test"],
"acl": ["*"],
}
indexd_client_accepting_record(
indexd_record_with_non_public_authz_and_public_acl_populated)
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
indexed_file = IndexedFile(file_id="some id")
with patch(
"fence.blueprints.data.indexd.IndexedFile.indexed_file_locations",
return_value=[],
):
with pytest.raises(NotFound):
indexed_file._get_signed_url(
protocol=supported_protocol,
action=supported_action,
expires_in=10,
force_signed_url=True,
r_pays_project=None,
file_name="some file",
)
def test_delete_files_successful(app, public_bucket_indexd_client):
"""
Test fence.blueprints.data.indexd.IndexedFile delete_files is successful
"""
class MockBlobServiceClient:
"""
Mock Blob Service Client
"""
def __init__(self, conn_str):
"""
Setup MockBlobServiceClient
"""
self.conn_str = conn_str
def get_blob_client(self, container_name, blob_name):
"""
Get a MockBlobClient
"""
return MockBlobClient(container_name=container_name,
blob_name=blob_name)
class MockBlobClient:
"""
Mock Blob Client
"""
def __init__(self, container_name, blob_name):
"""
Setup MockBlobClient
"""
self.container_name = container_name
self.blob_name = blob_name
def delete_blob(self):
"""
Delete a blob
"""
return
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.resources.user.user_session.UserSession.create_initial_token"
):
with patch(
"fence.blueprints.data.indexd.flask.current_app.boto.delete_data_file",
return_value=("", 204),
):
with patch("cirrus.GoogleCloudManager.delete_data_file",
return_value=("", 204)):
with patch(
"fence.blueprints.data.indexd.BlobServiceClient.from_connection_string",
return_value=MockBlobServiceClient(
conn_str="some_connection_string"),
):
indexed_file = IndexedFile(file_id="some id")
assert indexed_file.delete_files()
def test_indexed_file_index_document_request_has_json_no_urls(app):
"""
Test fence.blueprints.data.indexd.IndexedFile call to index_document with JSON without URLs
"""
class MockResponse:
"""
Mock response for requests lib
"""
def __init__(self, data, status_code=200):
"""
Setup mock response
"""
self.data = data
self.status_code = status_code
def json(self):
"""
Mock json() call
"""
return self.data
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.blueprints.data.indexd.requests.get",
return_value=MockResponse({"not_urls": ["some url"]}),
):
indexed_file = IndexedFile(file_id="some id")
with pytest.raises(InternalError):
indexed_file.index_document
def test_indexed_file_index_document_request_has_json(app, supported_protocol):
"""
Test fence.blueprints.data.indexd.IndexedFile call to index_document with JSON
"""
class MockResponse:
"""
Mock response for requests lib
"""
def __init__(self, data, status_code=200):
"""
Setup mock response
"""
self.data = data
self.status_code = status_code
def json(self):
"""
Mock json() call
"""
return self.data
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.blueprints.data.indexd.requests.get",
return_value=MockResponse(
{"urls": [f"{supported_protocol}://some/url"]}),
):
indexed_file = IndexedFile(file_id="some id")
assert indexed_file.index_document
def test_indexed_file_index_document_request_has_json_exception(
app, supported_protocol):
"""
Test fence.blueprints.data.indexd.IndexedFile call to index_document with JSON ValueError
"""
class MockResponse:
"""
Mock response for requests lib
"""
def __init__(self, data, status_code=200):
"""
Setup mock response
"""
self.data = data
self.status_code = status_code
def json(self):
"""
Mock json() call with ValueError
"""
raise ValueError("unable to get json")
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.blueprints.data.indexd.requests.get",
return_value=MockResponse(
{"urls": [f"{supported_protocol}://some/url"]}),
):
indexed_file = IndexedFile(file_id="some id")
with pytest.raises(InternalError):
indexed_file.index_document
def test_delete_files_fails_invalid_connection_string(
app, public_bucket_indexd_client):
"""
Test fence.blueprints.data.indexd.IndexedFile delete_files fails
because of an invalid connection string
"""
class MockBlobServiceClient:
"""
Mock Blob Service Client
"""
def __init__(self, conn_str):
"""
Setup MockBlobServiceClient
"""
self.conn_str = conn_str
@classmethod
def from_connection_string(cls, container_name, blob_name):
"""
Get a MockBlobClient
"""
raise ValueError("Connection string is either blank or malformed.")
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.resources.user.user_session.UserSession.create_initial_token"
):
with patch(
"fence.blueprints.data.indexd.flask.current_app.boto.delete_data_file",
side_effect=ValueError("Invalid connection string"),
):
with patch(
"cirrus.GoogleCloudManager.delete_data_file",
side_effect=ValueError("Invalid connection string"),
):
with patch(
"fence.blueprints.data.indexd.BlobServiceClient.from_connection_string",
return_value=MockBlobServiceClient(
conn_str="invalid connection string"),
):
indexed_file = IndexedFile(file_id="some id")
message, status_code = indexed_file.delete_files()
assert message == "Failed to delete data file."
assert status_code == 500
def test_get_authorized_with_username_missing_value_error(
app, supported_action, supported_protocol,
indexd_client_accepting_record):
"""
Test fence.blueprints.data.indexd.IndexedFile get_authorized_with_username without authz in indexd record
"""
indexd_record_with_no_authz_and_public_acl_populated = {
"urls": [f"{supported_protocol}://some/location"],
"noauthz": ["/programs/DEV/projects/test"],
"acl": [],
}
indexd_client_accepting_record(
indexd_record_with_no_authz_and_public_acl_populated)
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
indexed_file = IndexedFile(file_id="some id")
with pytest.raises(ValueError):
indexed_file.get_authorized_with_username(supported_action)
def test_internal_get_signed_url_no_protocol_index_error(
app, supported_action, supported_protocol,
indexd_client_accepting_record):
"""
Test fence.blueprints.data.indexd.IndexedFile call to get_signed_url
without a protocol gives NotFound error
"""
indexd_record_with_non_public_authz_and_public_acl_populated = {
"urls": [f"{supported_protocol}://some/location"],
"authz": ["/programs/DEV/projects/test"],
"acl": ["*"],
}
indexd_client_accepting_record(
indexd_record_with_non_public_authz_and_public_acl_populated)
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
indexed_file = IndexedFile(file_id="some id")
with patch(
"fence.blueprints.data.indexd.IndexedFileLocation.get_signed_url",
side_effect=IndexError(),
):
with patch(
"fence.blueprints.data.indexd.S3IndexedFileLocation.get_signed_url",
side_effect=IndexError(),
):
with patch(
"fence.blueprints.data.indexd.GoogleStorageIndexedFileLocation.get_signed_url",
side_effect=IndexError(),
):
with patch(
"fence.blueprints.data.indexd.AzureBlobStorageIndexedFileLocation.get_signed_url",
side_effect=IndexError(),
):
with pytest.raises(NotFound):
indexed_file._get_signed_url(
protocol=None,
action=supported_action,
expires_in=10,
force_signed_url=True,
r_pays_project=None,
file_name="some file",
)
def test_get_signed_url_action_not_supported(app, supported_protocol,
indexd_client_accepting_record):
"""
Test fence.blueprints.data.indexd.IndexedFile call to get_signed_url action not supported
"""
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
indexd_record_with_non_public_authz_and_public_acl_populated = {
"urls": [f"{supported_protocol}://some/location"],
"authz": None,
"acl": ["*"],
}
indexd_client_accepting_record(
indexd_record_with_non_public_authz_and_public_acl_populated)
indexed_file = IndexedFile(file_id="some id")
with pytest.raises(NotSupported):
assert indexed_file.get_signed_url(protocol=supported_protocol,
action="not_supported",
expires_in=10)
def test_indexed_file_index_document_request_not_available(app):
"""
Test fence.blueprints.data.indexd.IndexedFile call to index_document with Unavailable Error
"""
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.blueprints.data.indexd.requests.get",
side_effect=Exception("url not available"),
):
indexed_file = IndexedFile(file_id="some id")
with pytest.raises(UnavailableError):
print(indexed_file.index_document)
def delete_data_file(file_id):
"""
Delete all the locations for a data file which was uploaded to bucket storage from
indexd.
If the data file is still at the first stage where it belongs to just the uploader
(and isn't linked to a project), then the deleting user should match the uploader
field on the record in indexd. Otherwise, the user must have delete permissions in
the project.
Args:
file_id (str): GUID of file to delete
"""
record = IndexedFile(file_id)
# check auth: user must have uploaded the file (so `uploader` field on the record is
# this user)
uploader = record.index_document.get("uploader")
if not uploader:
raise Forbidden("deleting submitted records is not supported")
if current_token["context"]["user"]["name"] != uploader:
raise Forbidden("user is not uploader for file {}".format(file_id))
logger.info("deleting record and files for {}".format(file_id))
record.delete_files(delete_all=True)
return record.delete()
def test_set_acl_missing_unauthorized(app, supported_protocol,
indexd_client_accepting_record):
"""
Test fence.blueprints.data.indexd.IndexedFile set_acls as unauthorized from indexd record
"""
indexd_record_with_non_public_authz_and_no_public_acl_populated = {
"urls": [f"{supported_protocol}://some/location"],
"authz": ["/programs/DEV/projects/test"],
"noacl": [],
}
indexd_client_accepting_record(
indexd_record_with_non_public_authz_and_no_public_acl_populated)
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
indexed_file = IndexedFile(file_id="some id")
with pytest.raises(Unauthorized):
indexed_file.set_acls
def test_indexed_file_index_document_request_service_not_available(app):
"""
Test fence.blueprints.data.indexd.IndexedFile call to index_document service not available
"""
class MockResponse:
"""
Mock response for requests lib
"""
def __init__(self, data, status_code=503):
"""
Setup mock response 503
"""
self.data = data
self.status_code = status_code
def json(self):
"""
Mock json() call
"""
return self.data
def text(self):
"""
Mock text() call
"""
return "Not Found"
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.blueprints.data.indexd.requests.get",
return_value=MockResponse(data=None),
):
indexed_file = IndexedFile(file_id="some id")
with pytest.raises(UnavailableError):
indexed_file.index_document
def delete_data_file(file_id):
"""
Delete all the locations for a data file which was uploaded to bucket storage from
indexd.
If the data file has authz matching the user's permissions, delete it.
If the data file has no authz, then the deleting user should match the uploader
field on the record in indexd.
Args:
file_id (str): GUID of file to delete
"""
record = IndexedFile(file_id)
authz = record.index_document.get("authz")
has_correct_authz = None
if authz:
logger.debug(
"Trying to ask arborist if user can delete in fence for {}".format(authz)
)
has_correct_authz = flask.current_app.arborist.auth_request(
jwt=get_jwt(), service="fence", methods="delete", resources=authz
)
# If authz is not empty, use *only* arborist to check if user can delete
# Don't fall back on uploader -- this prevents users from escalating from edit to
# delete permissions by changing the uploader field to their own username
# (b/c users only have edit access through arborist/authz)
if has_correct_authz:
logger.info("Deleting record and files for {}".format(file_id))
message, status_code = record.delete_files(delete_all=True)
if str(status_code)[0] != "2":
return flask.jsonify({"message": message}), status_code
try:
return record.delete()
except Exception as e:
logger.error(e)
return (
flask.jsonify(
{"message": "There was an error deleting this index record."}
),
500,
)
else:
return (
flask.jsonify(
{
"message": "You do not have arborist permissions to delete this file."
}
),
403,
)
# If authz is empty: use uploader == user to see if user can delete.
uploader_mismatch_error_message = "You cannot delete this file because the uploader field indicates it does not belong to you."
uploader = record.index_document.get("uploader")
if not uploader:
return (
flask.jsonify({"message": uploader_mismatch_error_message}),
403,
)
if current_token["context"]["user"]["name"] != uploader:
return (
flask.jsonify({"message": uploader_mismatch_error_message}),
403,
)
logger.info("deleting record and files for {}".format(file_id))
message, status_code = record.delete_files(delete_all=True)
if str(status_code)[0] != "2":
return flask.jsonify({"message": message}), status_code
try:
return record.delete()
except Exception as e:
logger.error(e)
return (
flask.jsonify(
{"message": "There was an error deleting this index record."}
),
500,
)
def test_delete_files_unable_to_get_file_name(app,
public_bucket_indexd_client):
"""
Test fence.blueprints.data.indexd.IndexedFile delete_files with missing file name
"""
class MockBlobServiceClient:
"""
Mock Blob Service Client
"""
def __init__(self, conn_str):
"""
Setup MockBlobServiceClient
"""
self.conn_str = conn_str
def get_blob_client(self, container_name, blob_name):
"""
Get a MockBlobClient
"""
return MockBlobClient(container_name=container_name,
blob_name=blob_name)
class MockBlobClient:
"""
Mock Blob Client
"""
def __init__(self, container_name, blob_name):
"""
Setup MockBlobClient
"""
self.container_name = container_name
self.blob_name = blob_name
def delete_blob(self):
"""
Mock delete_blob to raise Exception
"""
raise Exception("url not available")
with patch("fence.blueprints.data.indexd.flask.current_app",
return_value=app):
with patch(
"fence.blueprints.data.indexd.S3IndexedFileLocation.file_name",
side_effect=Exception("url not available"),
):
with patch(
"fence.blueprints.data.indexd.GoogleStorageIndexedFileLocation.file_name",
side_effect=Exception("url not available"),
):
with patch(
"fence.blueprints.data.indexd.AzureBlobStorageIndexedFileLocation.file_name",
side_effect=Exception("url not available"),
):
with patch(
"fence.resources.user.user_session.UserSession.create_initial_token"
):
with patch(
"fence.blueprints.data.indexd.flask.current_app.boto.delete_data_file",
side_effect=Exception("url not available"),
):
with patch(
"cirrus.GoogleCloudManager.delete_data_file",
side_effect=Exception("url not available"),
):
with patch(
"fence.blueprints.data.indexd.BlobServiceClient.from_connection_string",
return_value=MockBlobServiceClient(
conn_str="some_connection_string"),
):
indexed_file = IndexedFile(
file_id="some id")
assert indexed_file.delete_files()
