def test_serialize_message():
tx = "AU"
rx = "CN"
s = str(uuid.uuid4())
t = str(uuid.uuid4())
p = str(uuid.uuid4())
msg = gd.Message(
sender=Jurisdiction(tx),
receiver=Jurisdiction(rx),
subject=URI(s),
obj=URI(t),
predicate=URI(p))
expected_json = """
{{
"sender": "{}",
"receiver": "{}",
"subject": "{}",
"obj": "{}",
"predicate": "{}"
}}
""".format(tx, rx, s, t, p)
msg_json = json.dumps(msg, cls=ser.MessageJSONEncoder)
assert json.loads(msg_json) == json.loads(expected_json)
def _generate_message_params():
msg_dict = _generate_msg_dict()
A = Jurisdiction(msg_dict["sender"])
B = Jurisdiction(msg_dict["receiver"])
subject = u.URI(msg_dict["subject"])
obj = u.URI(msg_dict["obj"])
predicate = u.URI(msg_dict["predicate"])
return (A, B, subject, obj, predicate)
def test_jurisdiction_object_initialization_failure():
# no argument
with pytest.raises(TypeError):
Jurisdiction()
# lowercase
with pytest.raises(TypeError):
Jurisdiction('us')
# more than 2 letters
with pytest.raises(TypeError):
Jurisdiction('USA')
# less than 2 letters
with pytest.raises(TypeError):
Jurisdiction('U')
# empty string
with pytest.raises(TypeError):
Jurisdiction('')
# not string argument
with pytest.raises(AssertionError):
Jurisdiction(1)
# unknown jurisdiction
with mock.patch('pycountry.countries.get',
return_value=None) as get_country:
with pytest.raises(ValueError):
Jurisdiction('US')
get_country.assert_called()
def process(self, job_id, job):
logger.info(
"[%s] Running the RetrieveAndStoreForeignDocumentsUseCase for job %s",
self.jurisdiction, job)
multihash = job['object']
sender = Jurisdiction(job['sender'])
# 1. check if this object is not in the object lake yet
# 2. if not - download it to the object lake
if not self._is_in_object_lake(multihash):
self._download_remote_obj(sender, multihash)
# 3. Give receiver access to the object
self.object_acl_repo.allow_access_to(multihash, self.jurisdiction.name)
# 4. Delete the job as completed
# 4.1. Schedule downloads of sub-documents
self.object_retrieval.delete(job_id)
return True
def test_get_document(ObjectLakeRepoMock, ObjectACLRepoMock, client):
# testing jurisdiction behaviour
search = ObjectACLRepoMock.return_value.search
search.return_value = [Jurisdiction(VALID_JURISDICTION_NAME)]
get_body = ObjectLakeRepoMock.return_value.get_body
get_body.return_value = json.dumps(DOCUMENT_JSON)
resp = client.get(DOCUMENT_GET_URL.format(VALID_DOCUMENT_URI),
headers=VALID_AUTH_HEADERS)
assert resp.mimetype == VALID_GET_RESPONSE_MIMETYPE, resp.data
assert resp.status_code == HTTPStatus.OK, resp.data
assert json.loads(resp.data) == DOCUMENT_JSON, resp.data
search.assert_called_once()
get_body.assert_called_once()
def search(self, filters=None):
"""
# TODO: we may do a faster version of it just to check if given jurisdiction
# may do this operation
Because post() stores each message
at the location representing the object,
with a name representing the recipient
(not implemented) (plus a timestamp, in case they received
messages about the same object more than once)
, we are able to scan that location
to see who has been granted access to the object.
"""
# if filters is None:
# return False
assert filters # shoudln't be used without filters, it's a programming error
# we only support one filter for now
allowed_filters = ('object__eq', )
for f in filters.keys():
if f not in allowed_filters:
# TODO ?
raise Exception("unsupported filter {} (not in {})".format(
f, allowed_filters))
found_objects = self.client.list_objects(
Bucket=self.bucket,
Prefix=miniorepo.slash_chunk(filters['object__eq']) + '/',
# recursive=True
)
if not found_objects.get('Contents', []):
return []
else:
uniq_jurisdictions = set()
for obj in found_objects.get('Contents', []):
pure_filename = obj['Key'].split('/')[-1]
if obj['Key'].endswith('.json'):
# based on rx_message, there is message in this file
# oname is something like /QmXx/xxx/xxxx/CN.json
jurisdiction_name = pure_filename.split('.')[0]
else:
# based on uploaded document, the file is empty
jurisdiction_name = pure_filename
uniq_jurisdictions.add(jurisdiction_name)
return [Jurisdiction(c) for c in uniq_jurisdictions]
def document_post(jurisdiction_name):
"""
---
post:
parameters:
- in: path
name: jurisdiction_name
required: true
schema:
type: string
requestBody:
content:
application/binary:
schema:
format: binary
type: string
responses:
200:
content:
application/json:
schema:
properties:
multihash:
format: uuid
type: string
type: object
description: Returns document id
"""
try:
target_jurisdiction = Jurisdiction(jurisdiction_name)
except Exception as e:
raise BadJurisdictionNameError(e)
object_lake_repo = ObjectLakeRepo(Config.OBJECT_LAKE_CONN)
object_acl_repo = ObjectACLRepo(Config.OBJECT_ACL_CONN)
if len(request.files) == 0:
raise NoInputFileError()
elif len(request.files) > 1:
raise TooManyFilesError(len(request.files))
# get the first file, whatever way it's called
file = request.files[list(request.files.keys())[0]]
use_case = StoreObjectUseCase(
object_acl_repo=object_acl_repo,
object_lake_repo=object_lake_repo,
)
try:
multihash = use_case.execute(fobj=file,
target_jurisdiction=target_jurisdiction)
except Exception as e:
logger.exception(e)
raise InternalServerError(e)
return Response(json.dumps({
"multihash": multihash,
}),
mimetype='application/json',
status=HTTPStatus.OK)
def document_fetch(uri):
"""
---
get:
parameters:
- in: path
name: uri
required: true
schema:
format: uuid
type: string
responses:
200:
content:
application/binary:
schema:
format: binary
type: string
description: Returns document
"""
if not URI(uri).is_valid_multihash():
raise InvalidURIError()
object_lake_repo = ObjectLakeRepo(Config.OBJECT_LAKE_CONN)
object_acl_repo = ObjectACLRepo(Config.OBJECT_ACL_CONN)
use_case = AuthenticatedObjectAccessUseCase(
object_acl_repo=object_acl_repo,
object_lake_repo=object_lake_repo,
)
request_auth = getattr(request, "auth", None)
if request_auth and 'jurisdiction' in request_auth:
try:
auth_jurisdiction = Jurisdiction(request_auth['jurisdiction'])
except Exception as e:
raise BadJurisdictionNameError(e)
else:
# no auth is provided, trust the GET request
# assuming JWT will handle it
# TODO: ensure that the auth provided allows access from that country
try:
auth_jurisdiction = Jurisdiction(request.args["as_jurisdiction"])
except Exception as e:
raise BadJurisdictionNameError(e)
try:
document_body = use_case.execute(uri, auth_jurisdiction)
except Exception as e:
logger.exception(e)
raise InternalServerError(e)
if document_body is not None:
return Response(
document_body,
status=HTTPStatus.OK,
mimetype='binary/octet-stream', # TODO: correct mimetype?
# TODO: some information about the file content?
)
else:
raise DocumentNotFoundError(uri, auth_jurisdiction)
def _prepare_use_case(self):
self.use_case = RetrieveAndStoreForeignDocumentsUseCase(
jurisdiction=Jurisdiction(env("IGL_JURISDICTION", default='AU')),
**self.repos)
def test_jurisdiction_object_initialization():
# For now I don't see any other way to check proper initialization
# initialization failure cases in this test are much more important
assert Jurisdiction('US').name == 'US'
def test_get_document_errors(ObjectLakeRepoMock, ObjectACLRepoMock, client):
# # testing unauthorized
# resp = client.get(DOCUMENT_GET_URL.format(VALID_DOCUMENT_URI))
# assert resp.status_code == HTTPStatus.UNAUTHORIZED
# testing invalid URI error
resp = client.get(DOCUMENT_GET_URL.format(INVALID_DOCUMENT_URI),
headers=VALID_AUTH_HEADERS)
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE, resp.data
assert resp.status_code == HTTPStatus.BAD_REQUEST, resp.data
assert resp.get_json() == error_response_json_template(InvalidURIError())
# testing invalid jurisdiction auth headers
try:
Jurisdiction(INVALID_JURISDICTION_NAME)
except Exception as e:
jurisdiction_exception = e
resp = client.get(DOCUMENT_GET_URL.format(VALID_DOCUMENT_URI),
headers=INVALID_AUTH_HEADERS)
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE, resp.data
assert resp.status_code == HTTPStatus.BAD_REQUEST, resp.data
assert resp.get_json() == error_response_json_template(
BadJurisdictionNameError(jurisdiction_exception))
search = ObjectACLRepoMock.return_value.search
get_body = ObjectLakeRepoMock.return_value.get_body
# testing not authenticated user
search.return_value = []
resp = client.get(DOCUMENT_GET_URL.format(VALID_DOCUMENT_URI),
headers=VALID_AUTH_HEADERS)
search.assert_called_once()
get_body.assert_not_called()
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE, resp.data
assert resp.status_code == HTTPStatus.NOT_FOUND, resp.data
assert resp.get_json() == error_response_json_template(
DocumentNotFoundError(VALID_DOCUMENT_URI,
Jurisdiction(VALID_JURISDICTION_NAME)))
# testing unexpected acl repo error
search.reset_mock()
get_body.reset_mock()
search.side_effect = Exception('Very bad times indeed')
resp = client.get(DOCUMENT_GET_URL.format(VALID_DOCUMENT_URI),
headers=VALID_AUTH_HEADERS)
search.assert_called_once()
get_body.assert_not_called()
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE, resp.data
assert resp.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, resp.data
assert resp.get_json() == error_response_json_template(
InternalServerError(search.side_effect))
# testing expected lake repo error
class NoSuchKey(Exception):
pass
search.reset_mock()
get_body.reset_mock()
search.side_effect = None
search.return_value = [Jurisdiction(VALID_JURISDICTION_NAME)]
get_body.side_effect = NoSuchKey
resp = client.get(DOCUMENT_GET_URL.format(VALID_DOCUMENT_URI),
headers=VALID_AUTH_HEADERS)
search.assert_called_once()
get_body.assert_called_once()
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE, resp.data
assert resp.status_code == HTTPStatus.NOT_FOUND, resp.data
assert resp.get_json() == error_response_json_template(
DocumentNotFoundError(VALID_DOCUMENT_URI,
Jurisdiction(VALID_JURISDICTION_NAME)))
# testing get_body return None
# Can it even happen? Or we should consider NoSuchKey
# error to be only way to get None as a result
# of the use case?
search.reset_mock()
get_body.reset_mock()
search.return_value = [Jurisdiction(VALID_JURISDICTION_NAME)]
get_body.side_effect = None
get_body.return_value = None
resp = client.get(DOCUMENT_GET_URL.format(VALID_DOCUMENT_URI),
headers=VALID_AUTH_HEADERS)
search.assert_called_once()
get_body.assert_called_once()
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE, resp.data
assert resp.status_code == HTTPStatus.NOT_FOUND, resp.data
assert resp.get_json() == error_response_json_template(
DocumentNotFoundError(VALID_DOCUMENT_URI,
Jurisdiction(VALID_JURISDICTION_NAME)))
# testing unexpected lake repo error
search.reset_mock()
get_body.reset_mock()
search.return_value = [Jurisdiction(VALID_JURISDICTION_NAME)]
get_body.side_effect = Exception('Bad times came')
resp = client.get(DOCUMENT_GET_URL.format(VALID_DOCUMENT_URI),
headers=VALID_AUTH_HEADERS)
search.assert_called_once()
get_body.assert_called_once()
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE, resp.data
assert resp.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, resp.data
assert resp.get_json() == error_response_json_template(
InternalServerError(get_body.side_effect))
def test_post_document_errors(ObjectLakeRepoMock, ObjectACLRepoMock, client):
resp = client.post(INVALID_POST_REQUEST_JURISDICTION_URL,
mimetype=INVALID_POST_REQUEST_MIMETYPE,
data={})
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE
assert resp.status_code == HTTPStatus.UNSUPPORTED_MEDIA_TYPE
assert resp.get_json() == error_response_json_template(
UnsupportedMediaTypeError(INVALID_POST_REQUEST_MIMETYPE,
[VALID_POST_REQUEST_MIMETYPE], []))
try:
Jurisdiction(INVALID_JURISDICTION_NAME)
except Exception as e:
jurisdiction_exception = e
resp = client.post(INVALID_POST_REQUEST_JURISDICTION_URL,
mimetype=VALID_POST_REQUEST_MIMETYPE,
data={})
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE
assert resp.status_code == HTTPStatus.BAD_REQUEST
assert resp.get_json() == error_response_json_template(
BadJurisdictionNameError(jurisdiction_exception))
resp = client.post(VALID_POST_REQUEST_JURISDICTION_URL,
mimetype=VALID_POST_REQUEST_MIMETYPE,
data={})
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE
assert resp.status_code == HTTPStatus.BAD_REQUEST
assert resp.get_json() == error_response_json_template(NoInputFileError())
resp = client.post(VALID_POST_REQUEST_JURISDICTION_URL,
mimetype=VALID_POST_REQUEST_MIMETYPE,
data=get_too_many_post_request_files())
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE
assert resp.status_code == HTTPStatus.BAD_REQUEST
assert resp.get_json() == error_response_json_template(
TooManyFilesError(2))
allow_access_to = ObjectACLRepoMock.return_value.allow_access_to
post_from_file_obj = ObjectLakeRepoMock.return_value.post_from_file_obj
allow_access_to.side_effect = Exception('Very bad thing: ACL')
post_from_file_obj.side_effect = Exception('Very bad thing: Lake')
resp = client.post(VALID_POST_REQUEST_JURISDICTION_URL,
mimetype=VALID_POST_REQUEST_MIMETYPE,
data=get_valid_post_request_files())
allow_access_to.assert_called_once()
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE
assert resp.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
assert resp.get_json() == error_response_json_template(
InternalServerError(allow_access_to.side_effect))
allow_access_to.reset_mock()
allow_access_to.side_effect = None
resp = client.post(VALID_POST_REQUEST_JURISDICTION_URL,
mimetype=VALID_POST_REQUEST_MIMETYPE,
data=get_valid_post_request_files())
allow_access_to.assert_called_once()
post_from_file_obj.assert_called_once()
assert resp.mimetype == VALID_ERROR_RESPONSE_MIMETYPE
assert resp.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
assert resp.get_json() == error_response_json_template(
InternalServerError(post_from_file_obj.side_effect))