def test_not_found(self, mocker):
mocked_logger = mocker.patch("core.tasks.logger")
obj_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
download_data_source(obj_id, model=self.model)
mocked_logger.info.assert_called_once_with(
"Datasource %s %s not found" % (self.model, obj_id),
extra={
"MESSAGE_ID": "datasource_not_found",
"MODEL": self.model,
"DATASOURCE_ID": obj_id,
"TASK": "download_data_source",
},
)
def test_fail_request_on_data_download(self, mocked_request, url_obj):
response = Response(status_code=403)
mocked_request.get.side_effect = [response]
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == Url.QUEUED_DOWNLOAD
assert not url_obj.downloaded
download_data_source(url_obj.id, model=self.model)
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == "failed"
assert url_obj.error == f"{response.status_code}: {response.reason}"
assert mocked_request.get.call_count == 1
def test_no_left_space(self, mocked_request, url_obj, dataset, mocker):
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == Url.QUEUED_DOWNLOAD
assert not url_obj.downloaded
mocked_request.get.return_value.iter_content = mocker.MagicMock()
mocked_request.get.return_value.iter_content.side_effect = OSError(
errno.ENOSPC, "No left space.")
download_data_source(url_obj.id, model=self.model)
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == Url.FAILED
assert url_obj.error == "Currently, the space limit was reached. Please try again later."
def test_task_download_data_source(self, event_loop, url_obj,
mocked_request):
application = URLRouter([
re_path(r"ws/api/(?P<upload_id>[0-9a-f-]+)/$",
ValidationConsumer.as_asgi())
])
communicator = WebsocketCommunicator(application,
f"/ws/api/{url_obj.id}/")
event_loop.run_until_complete(communicator.connect())
download_data_source(str(url_obj.id), model="Url")
message = event_loop.run_until_complete(communicator.receive_from())
event_loop.run_until_complete(communicator.disconnect())
message = json.loads(message)
assert message["type"] == "task.download_data_source"
assert message["datasource"]["id"] == str(url_obj.id)
assert message["datasource"]["status"] == "downloading"
assert not message["datasource"]["downloaded"]
def test_fail_request_on_analyzed_data_download(self, mocked_request,
url_obj, dataset):
response = Response(status_code=403)
success_response = mocked_request.get.return_value
mocked_request.get.side_effect = [success_response, response]
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == Url.QUEUED_DOWNLOAD
assert not url_obj.downloaded
download_data_source(url_obj.id, model=self.model)
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == "failed"
assert url_obj.error == f"{response.status_code}: {response.reason}"
assert mocked_request.get.call_count == 2
with open(url_obj.file.path) as f:
data = json.loads(f.read())
assert data == json.loads(dataset.read())
def test_success(self, mocked_request, url_obj, dataset):
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == Url.QUEUED_DOWNLOAD
assert not url_obj.downloaded
download_data_source(url_obj.id, model=self.model)
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == Url.QUEUED_VALIDATION
assert url_obj.downloaded
test_dataset = json.loads(dataset.read())
with open(url_obj.file.path) as f:
data = json.loads(f.read())
assert data == test_dataset
with open(url_obj.analyzed_file.path) as f:
data = json.loads(f.read())
assert data == test_dataset
def test_exception_on_analyzed_data_download(self, mocked_request, url_obj,
dataset):
success_response = mocked_request.get.return_value
mocked_request.get.side_effect = [
success_response,
Exception("Some error from remote host")
]
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == Url.QUEUED_DOWNLOAD
assert not url_obj.downloaded
download_data_source(url_obj.id, model=self.model)
url_obj = Url.objects.get(id=url_obj.id)
assert url_obj.status == "failed"
assert url_obj.error == "Something went wrong. Contact with support service."
assert mocked_request.get.call_count == 2
with open(url_obj.file.path) as f:
data = json.loads(f.read())
assert data == json.loads(dataset.read())
def test_unregistered_model(self, mocked_request, url_obj):
download_data_source(url_obj.id, model="SomeNew")
assert mocked_request.get.call_count == 0