def test_releaseworkspaceapi_get_with_permission(api_rf, build_release_with_files):
workspace = WorkspaceFactory()
backend1 = BackendFactory(slug="backend1")
backend2 = BackendFactory(slug="backend2")
user = UserFactory()
ProjectMembershipFactory(
user=user, project=workspace.project, roles=[ProjectCollaborator]
)
# two release for same filename but different content
release1 = build_release_with_files(
["file1.txt"], workspace=workspace, backend=backend1, created_by=user
)
rfile1 = release1.files.first()
release2 = build_release_with_files(
["file1.txt"], workspace=workspace, backend=backend2, created_by=user
)
rfile2 = release2.files.first()
request = api_rf.get("/")
request.user = user
response = ReleaseWorkspaceAPI.as_view()(request, workspace_name=workspace.name)
assert response.status_code == 200
assert response.data == {
"files": [
{
"name": "backend2/file1.txt",
"id": rfile2.pk,
"url": f"/api/v2/releases/file/{rfile2.id}",
"user": rfile2.created_by.username,
"date": rfile2.created_at.isoformat(),
"size": rfile2.size,
"sha256": rfile2.filehash,
"is_deleted": False,
"backend": release2.backend.name,
"metadata": None,
"review": None,
},
{
"name": "backend1/file1.txt",
"id": rfile1.pk,
"url": f"/api/v2/releases/file/{rfile1.id}",
"user": rfile1.created_by.username,
"date": rfile1.created_at.isoformat(),
"size": rfile1.size,
"sha256": rfile1.filehash,
"is_deleted": False,
"backend": release1.backend.name,
"metadata": None,
"review": None,
},
],
}
def test_workspace_files_many_releases(freezer, build_release_with_files):
now = timezone.now()
backend1 = BackendFactory(slug="backend1")
backend2 = BackendFactory(slug="backend2")
workspace = WorkspaceFactory()
build_release_with_files(
["test1", "test2", "test3"],
workspace=workspace,
backend=backend1,
created_at=minutes_ago(now, 10),
)
build_release_with_files(
["test2", "test3"],
workspace=workspace,
backend=backend1,
created_at=minutes_ago(now, 8),
)
release3 = build_release_with_files(
["test2"],
workspace=workspace,
backend=backend1,
created_at=minutes_ago(now, 6),
)
release4 = build_release_with_files(
["test1", "test3"],
workspace=workspace,
backend=backend1,
created_at=minutes_ago(now, 4),
)
release5 = build_release_with_files(
["test1"],
workspace=workspace,
backend=backend1,
created_at=minutes_ago(now, 2),
)
# different backend, same file name, more recent
release6 = build_release_with_files(
["test1"],
workspace=workspace,
backend=backend2,
created_at=minutes_ago(now, 1),
)
output = releases.workspace_files(workspace)
assert output == {
"backend1/test1": release5.files.get(name="test1"),
"backend1/test3": release4.files.get(name="test3"),
"backend1/test2": release3.files.get(name="test2"),
"backend2/test1": release6.files.get(name="test1"),
}
def test_jobapiupdate_post_with_flags(api_rf):
backend = BackendFactory()
job_request = JobRequestFactory()
now = timezone.now()
data = [{
"identifier": "job1",
"job_request_id": job_request.identifier,
"action": "test-action",
"status": "running",
"status_code": "",
"status_message": "",
"created_at": minutes_ago(now, 2),
"started_at": minutes_ago(now, 1),
"updated_at": now,
"completed_at": None,
}]
flags = json.dumps(
{
"mode": {
"v": "test",
"ts": "1659092411.5025"
},
"paused": {
"v": " ",
"ts": "1657099752.16788"
},
"db-maintenance": {
"v": "",
"ts": "1657194377.72893"
},
},
separators=(",", ":"),
)
request = api_rf.post(
"/",
data=data,
format="json",
HTTP_AUTHORIZATION=backend.auth_token,
HTTP_FLAGS=flags,
)
response = JobAPIUpdate.as_view()(request)
assert response.status_code == 200, response.data
assert Job.objects.count() == 1
backend.refresh_from_db()
assert backend.jobrunner_state["mode"]["v"] == "test"
def test_releaseapi_post_success(api_rf, slack_messages, build_release, file_content):
creating_user = UserFactory()
uploading_user = UserFactory(roles=[OutputChecker])
backend = BackendFactory(name="test-backend")
release = build_release(["file.txt"], backend=backend, created_by=creating_user)
BackendMembershipFactory(backend=release.backend, user=creating_user)
BackendMembershipFactory(backend=release.backend, user=uploading_user)
request = api_rf.post(
"/",
content_type="application/octet-stream",
data=file_content,
HTTP_CONTENT_DISPOSITION="attachment; filename=file.txt",
HTTP_AUTHORIZATION=release.backend.auth_token,
HTTP_OS_USER=uploading_user.username,
)
response = ReleaseAPI.as_view()(request, release_id=release.id)
rfile = release.files.first()
assert response.status_code == 201, response.data
assert response.headers["Location"].endswith(f"/releases/file/{rfile.id}")
assert response.headers["File-Id"] == rfile.id
assert len(slack_messages) == 1
text, channel = slack_messages[0]
assert channel == "opensafely-releases"
assert f"{uploading_user.get_staff_url()}|{uploading_user.name}>" in text
assert f"{release.get_absolute_url()}|release>" in text
assert f"{rfile.get_absolute_url()}|{rfile.name}>" in text
assert release.backend.name in text
def test_userapidetail_unknown_user(api_rf):
backend = BackendFactory()
request = api_rf.get("/", HTTP_AUTHORIZATION=backend.auth_token)
response = UserAPIDetail.as_view()(request, username="******")
assert response.status_code == 404
def test_create_release_success():
backend = BackendFactory()
workspace = WorkspaceFactory()
user = UserFactory()
files = {"file1.txt": "hash"}
release = releases.create_release(workspace, backend, user, files)
assert release.requested_files == files
def test_jobrequestapicreate_success(api_rf, pipeline_config):
backend = BackendFactory()
workspace = WorkspaceFactory()
user = UserFactory()
token = user.rotate_token()
assert not JobRequest.objects.exists()
data = {
"workspace": workspace.name,
"backend": backend.slug,
"sha": "123",
"project_definition": pipeline_config,
"requested_actions": ["test"],
}
request = api_rf.post("/",
HTTP_AUTHORIZATION=f"{user.username}:{token}",
data=data)
response = JobRequestAPICreate.as_view()(request)
assert response.status_code == 201, response.data
assert JobRequest.objects.count() == 1
job_request = JobRequest.objects.first()
assert job_request.workspace == workspace
def test_jobapiupdate_unknown_job_request(api_rf):
backend = BackendFactory()
JobRequestFactory()
now = timezone.now()
data = [{
"identifier": "job1",
"job_request_id": "test",
"action": "test-action",
"status": "running",
"status_code": "",
"status_message": "",
"created_at": minutes_ago(now, 2),
"started_at": minutes_ago(now, 1),
"updated_at": now,
"completed_at": None,
}]
request = api_rf.post("/",
HTTP_AUTHORIZATION=backend.auth_token,
data=data,
format="json")
response = JobAPIUpdate.as_view()(request)
assert response.status_code == 400, response.data
assert "Unknown JobRequest IDs" in response.data[0]
def test_releaseworkspaceapi_post_with_bad_backend_token(api_rf):
workspace = WorkspaceFactory()
BackendFactory(auth_token="test")
request = api_rf.post("/", HTTP_AUTHORIZATION="invalid")
response = ReleaseWorkspaceAPI.as_view()(request, workspace_name=workspace.name)
assert response.status_code == 403
def test_update_stats_new_url():
backend = BackendFactory()
StatsFactory(backend=backend, url="test")
update_stats(backend, url="new-url")
# check there's only one Stats for backend
assert backend.stats.count() == 2
assert backend.stats.last().url == "new-url"
def test_update_stats_existing_url():
backend = BackendFactory()
StatsFactory(backend=backend, url="test")
update_stats(backend, url="test")
# check there's only one Stats for backend
assert backend.stats.count() == 1
assert backend.stats.first().url == "test"
def test_jobrequestapilist_produce_stats_when_authed(api_rf):
backend = BackendFactory()
assert Stats.objects.filter(backend=backend).count() == 0
request = api_rf.get("/", HTTP_AUTHORIZATION=backend.auth_token)
response = JobRequestAPIList.as_view()(request)
assert response.status_code == 200
assert Stats.objects.filter(backend=backend).count() == 1
def test_jobrequestapilist_filter_by_backend(api_rf):
backend = BackendFactory()
JobRequestFactory(backend=backend)
JobRequestFactory()
request = api_rf.get("/", HTTP_AUTHORIZATION=backend.auth_token)
response = JobRequestAPIList.as_view()(request)
assert response.status_code == 200, response.data
assert len(response.data["results"]) == 1
def test_build_hatch_token_and_url_default():
backend = BackendFactory()
user = UserFactory()
workspace = WorkspaceFactory()
releases.build_hatch_token_and_url(
backend=backend,
workspace=workspace,
user=user,
)
def test_validate_upload_access_unknown_user(rf):
backend = BackendFactory()
workspace = WorkspaceFactory()
BackendMembershipFactory(backend=backend)
request = rf.get("/", HTTP_AUTHORIZATION=backend.auth_token, HTTP_OS_USER="******")
with pytest.raises(NotAuthenticated):
validate_upload_access(request, workspace)
def test_jobapiupdate_all_new(api_rf):
backend = BackendFactory()
job_request = JobRequestFactory()
now = timezone.now()
assert Job.objects.count() == 0
data = [
{
"identifier": "job1",
"job_request_id": job_request.identifier,
"action": "test-action",
"status": "running",
"status_code": "",
"status_message": "",
"created_at": minutes_ago(now, 2),
"started_at": minutes_ago(now, 1),
"updated_at": now,
"completed_at": None,
},
{
"identifier": "job2",
"action": "test-action",
"job_request_id": job_request.identifier,
"status": "pending",
"status_code": "",
"status_message": "",
"created_at": minutes_ago(now, 2),
"updated_at": now,
"started_at": None,
"completed_at": None,
},
{
"identifier": "job3",
"job_request_id": job_request.identifier,
"action": "test-action",
"status": "running",
"status_code": "",
"status_message": "",
"created_at": minutes_ago(now, 2),
"started_at": None,
"updated_at": now,
"completed_at": None,
},
]
request = api_rf.post("/",
HTTP_AUTHORIZATION=backend.auth_token,
data=data,
format="json")
response = JobAPIUpdate.as_view()(request)
assert response.status_code == 200, response.data
assert Job.objects.count() == 3
def test_build_hatch_token_and_url_with_custom_expires():
backend = BackendFactory()
user = UserFactory()
workspace = WorkspaceFactory()
releases.build_hatch_token_and_url(
backend=backend,
workspace=workspace,
user=user,
expiry=timezone.now() + timedelta(minutes=3),
)
def test_userapidetail_success(api_rf):
backend = BackendFactory()
org = OrgFactory()
project = ProjectFactory(org=org)
user = UserFactory(roles=[CoreDeveloper])
OrgMembershipFactory(org=org, user=user, roles=[OrgCoordinator])
ProjectMembershipFactory(project=project,
user=user,
roles=[ProjectDeveloper])
request = api_rf.get("/", HTTP_AUTHORIZATION=backend.auth_token)
response = UserAPIDetail.as_view()(request, username=user.username)
assert response.status_code == 200
# permissions
permissions = response.data["permissions"]
assert permissions["global"] == [
"application_manage",
"backend_manage",
"org_create",
"user_manage",
]
assert permissions["orgs"] == [
# we have no permissions for OrgCoordinator yet
{
"slug": org.slug,
"permissions": [],
},
]
assert permissions["projects"] == [{
"slug":
project.slug,
"permissions": [
"job_cancel",
"job_run",
"snapshot_create",
"workspace_archive",
"workspace_create",
"workspace_toggle_notifications",
],
}]
# roles
roles = response.data["roles"]
assert roles["global"] == ["CoreDeveloper"]
assert roles["orgs"] == [{"slug": org.slug, "roles": ["OrgCoordinator"]}]
assert roles["projects"] == [{
"slug": project.slug,
"roles": ["ProjectDeveloper"]
}]
def test_validate_upload_access_not_a_backend_member(rf):
backend = BackendFactory()
user = UserFactory(roles=[OutputChecker])
workspace = WorkspaceFactory()
request = rf.get(
"/",
HTTP_AUTHORIZATION=backend.auth_token,
HTTP_OS_USER=user.username,
)
with pytest.raises(NotAuthenticated):
validate_upload_access(request, workspace)
def test_releaseworkspaceapi_post_create_release(api_rf, slack_messages):
user = UserFactory(roles=[OutputChecker])
workspace = WorkspaceFactory()
ProjectMembershipFactory(user=user, project=workspace.project)
backend = BackendFactory(auth_token="test", name="test-backend")
BackendMembershipFactory(backend=backend, user=user)
assert Release.objects.count() == 0
data = {
"files": [
{
"name": "file1.txt",
"url": "url",
"size": 7,
"sha256": "hash",
"date": timezone.now(),
"metadata": {},
"review": None,
}
],
"metadata": {},
"review": None,
}
request = api_rf.post(
"/",
data=data,
format="json",
HTTP_AUTHORIZATION="test",
HTTP_OS_USER=user.username,
)
response = ReleaseWorkspaceAPI.as_view()(request, workspace_name=workspace.name)
assert response.status_code == 201, response.data
assert Release.objects.count() == 1
release = Release.objects.first()
assert response["Release-Id"] == str(release.id)
assert response["Location"] == f"http://testserver{release.get_api_url()}"
assert len(slack_messages) == 1
text, channel = slack_messages[0]
assert channel == "opensafely-releases"
assert f"{user.get_staff_url()}|{user.name}>" in text
assert f"{release.get_absolute_url()}|release>" in text
assert f"{workspace.get_absolute_url()}|{workspace.name}>" in text
assert backend.name in text
def test_validate_upload_access_no_permission(rf):
backend = BackendFactory()
user = UserFactory()
workspace = WorkspaceFactory()
BackendMembershipFactory(backend=backend, user=user)
request = rf.get(
"/",
HTTP_AUTHORIZATION=backend.auth_token,
HTTP_OS_USER=user.username,
)
with pytest.raises(NotAuthenticated):
validate_upload_access(request, workspace)
def test_jobapiupdate_post_only(api_rf):
backend = BackendFactory()
# GET
request = api_rf.get("/", HTTP_AUTHORIZATION=backend.auth_token)
assert JobAPIUpdate.as_view()(request).status_code == 405
# HEAD
request = api_rf.head("/", HTTP_AUTHORIZATION=backend.auth_token)
assert JobAPIUpdate.as_view()(request).status_code == 405
# PATCH
request = api_rf.patch("/", HTTP_AUTHORIZATION=backend.auth_token)
assert JobAPIUpdate.as_view()(request).status_code == 405
# PUT
request = api_rf.put("/", HTTP_AUTHORIZATION=backend.auth_token)
assert JobAPIUpdate.as_view()(request).status_code == 405
def test_workspace_files_one_release(build_release):
backend = BackendFactory(slug="backend")
names = ["test1", "test2", "test3"]
release = build_release(names, backend=backend)
for name in names:
ReleaseFileFactory(
release=release,
workspace=release.workspace,
name=name,
)
output = releases.workspace_files(release.workspace)
assert output == {
"backend/test1": release.files.get(name="test1"),
"backend/test2": release.files.get(name="test2"),
"backend/test3": release.files.get(name="test3"),
}
def test_releasenotificationapicreate_success(api_rf, slack_messages):
backend = BackendFactory(name="test")
data = {
"created_by": "test user",
"path": "/path/to/outputs",
}
request = api_rf.post("/", data, HTTP_AUTHORIZATION=backend.auth_token)
request.user = UserFactory()
response = ReleaseNotificationAPICreate.as_view()(request)
assert response.status_code == 201, response.data
# check we called the slack API in the expected way
assert len(slack_messages) == 1
text, channel = slack_messages[0]
assert channel == "opensafely-releases"
assert text == f"test user released outputs from /path/to/outputs on {backend.name}"
def test_jobapiupdate_invalid_payload(api_rf):
backend = BackendFactory()
assert Job.objects.count() == 0
data = [{"action": "test-action"}]
request = api_rf.post("/",
HTTP_AUTHORIZATION=backend.auth_token,
data=data,
format="json")
response = JobAPIUpdate.as_view()(request)
assert Job.objects.count() == 0
assert response.status_code == 400, response.data
errors = response.data[0]
assert len(errors.keys()) == 9
def test_releaseworkspaceapi_post_with_bad_json(api_rf):
user = UserFactory(roles=[OutputChecker])
workspace = WorkspaceFactory()
ProjectMembershipFactory(user=user, project=workspace.project)
backend = BackendFactory(auth_token="test")
BackendMembershipFactory(backend=backend, user=user)
request = api_rf.post(
"/",
content_type="application/json",
data=json.dumps({}),
HTTP_CONTENT_DISPOSITION="attachment; filename=release.zip",
HTTP_AUTHORIZATION="test",
HTTP_OS_USER=user.username,
)
response = ReleaseWorkspaceAPI.as_view()(request, workspace_name=workspace.name)
assert response.status_code == 400
def test_jobapiupdate_post_with_errors(api_rf, mocker, error_message):
backend = BackendFactory()
job_request = JobRequestFactory()
mocked_sentry = mocker.patch("jobserver.api.jobs.sentry_sdk",
autospec=True)
now = timezone.now()
data = [
{
"identifier": "job",
"job_request_id": job_request.identifier,
"action": "test-action",
"status": "running",
"status_code": "",
"status_message": "Running",
"created_at": now,
"started_at": now,
"updated_at": now,
"completed_at": None,
},
]
request_1 = api_rf.post("/",
HTTP_AUTHORIZATION=backend.auth_token,
data=data,
format="json")
JobAPIUpdate.as_view()(request_1)
data[0]["status"] = "failed"
data[0]["status_message"] = error_message
request_2 = api_rf.post("/",
HTTP_AUTHORIZATION=backend.auth_token,
data=data,
format="json")
response = JobAPIUpdate.as_view()(request_2)
assert response.status_code == 200, response.data
mocked_sentry.capture_message.assert_called_once()
def test_create_release_new_style_success():
backend = BackendFactory()
workspace = WorkspaceFactory()
user = UserFactory()
files = [{
"name": "file1.txt",
"path": "path/to/file1.txt",
"url": "",
"size": 7,
"sha256": "hash",
"date": "2022-08-17T13:37Z",
"metadata": {},
}]
release = releases.create_release(workspace, backend, user, files)
assert release.requested_files == files
assert release.files.count() == 1
rfile = release.files.first()
rfile.filehash == "hash"
rfile.size == 7
def test_releaseapi_post_bad_backend(api_rf, build_release, file_content):
user = UserFactory(roles=[OutputChecker])
release = build_release(["output/file.txt"])
bad_backend = BackendFactory()
BackendMembershipFactory(backend=bad_backend, user=user)
content = "".join(random.choice(string.ascii_letters) for i in range(10))
content = content.encode("utf8")
request = api_rf.post(
"/",
content_type="application/octet-stream",
data=file_content,
HTTP_CONTENT_DISPOSITION="attachment; filename=output/file.txt",
HTTP_AUTHORIZATION=bad_backend.auth_token,
HTTP_OS_USER=user.username,
)
response = ReleaseAPI.as_view()(request, release_id=release.id)
assert response.status_code == 400
assert bad_backend.slug in response.data["detail"]
def test_jobapiupdate_all_existing(api_rf, freezer):
backend = BackendFactory()
job_request = JobRequestFactory()
now = timezone.now()
# 3 pending jobs already exist
job1, job2, job3, = JobFactory.create_batch(
3,
job_request=job_request,
started_at=None,
status="pending",
completed_at=None,
)
job1.identifier = "job1"
job1.save()
job2.identifier = "job2"
job2.save()
job3.identifier = "job3"
job3.save()
assert Job.objects.count() == 3
data = [
{
"identifier": "job1",
"job_request_id": job_request.identifier,
"action": "test-action1",
"status": "succeeded",
"status_code": "",
"status_message": "",
"created_at": minutes_ago(now, 2),
"started_at": minutes_ago(now, 1),
"updated_at": now,
"completed_at": seconds_ago(now, 30),
},
{
"identifier": "job2",
"job_request_id": job_request.identifier,
"action": "test-action2",
"status": "running",
"status_code": "",
"status_message": "",
"created_at": minutes_ago(now, 2),
"started_at": minutes_ago(now, 1),
"updated_at": now,
"completed_at": None,
},
{
"identifier": "job3",
"job_request_id": job_request.identifier,
"action": "test-action3",
"status": "pending",
"status_code": "",
"status_message": "",
"created_at": minutes_ago(now, 2),
"started_at": None,
"updated_at": now,
"completed_at": None,
},
]
request = api_rf.post("/",
HTTP_AUTHORIZATION=backend.auth_token,
data=data,
format="json")
response = JobAPIUpdate.as_view()(request)
assert response.status_code == 200, response.data
# we shouldn't have a different number of jobs
jobs = Job.objects.all()
assert len(jobs) == 3
# check our jobs look as expected
job1, job2, job3 = jobs
# succeeded
assert job1.identifier == "job1"
assert job1.started_at == minutes_ago(now, 1)
assert job1.updated_at == now
assert job1.completed_at == seconds_ago(now, 30)
# running
assert job2.identifier == "job2"
assert job2.started_at == minutes_ago(now, 1)
assert job2.updated_at == now
assert job2.completed_at is None
# pending
assert job3.identifier == "job3"
assert job3.started_at is None
assert job3.updated_at == now
assert job3.completed_at is None