def test_process_tasks(app, mocker):
"""Test task hanging."""
send_email = mocker.patch("orcid_hub.utils.send_email")
org = Organisation.select().first()
Task.insert_many(
dict(org=org,
updated_at=utils.datetime.utcnow(),
filename=tt.name,
task_type=tt.value) for tt in TaskType).execute()
utils.process_tasks()
send_email.assert_not_called()
Task.insert_many(
dict(org=org,
created_by=org.tech_contact,
created_at=utils.datetime(2017, 1, 1),
updated_at=utils.datetime(2017, 1, 1),
filename=tt.name,
task_type=tt.value) for tt in TaskType).execute()
utils.process_tasks()
send_email.assert_called()
task_count = Task.select().count()
utils.process_tasks()
assert Task.select().count() == task_count // 2
def test_test_database(models):
"""Test of the consitency of the test database."""
assert Organisation.select().count() == 10
assert User.select().count() == 63
assert OrcidToken.select().count() == 60
assert AffiliationRecord.select().count() == 10
assert FundingRecord.select().count() == 10
assert FundingContributor.select().count() == 10
assert FundingInvitee.select().count() == 10
assert ExternalId.select().count() == 10
assert WorkRecord.select().count() == 10
assert WorkContributor.select().count() == 10
assert WorkExternalId.select().count() == 10
assert WorkInvitee.select().count() == 10
assert PeerReviewRecord.select().count() == 10
assert PeerReviewExternalId.select().count() == 10
assert PeerReviewInvitee.select().count() == 10
assert ResearcherUrlRecord.select().count() == 10
assert OtherNameRecord.select().count() == 10
assert KeywordRecord.select().count() == 10
assert Task.select().count() == 30
assert UserOrgAffiliation.select().count() == 30
assert User.get(id=43).admin_for.count() == 10
assert User.get(id=1).admin_for.count() == 0
assert User.get(id=42).admin_for.count() > 0
assert User.get(id=2).organisations.count() > 0
assert Organisation.get(id=1).admins.count() == 1
assert Organisation.get(id=5).users.count() > 0
assert Organisation.get(id=5).admins.count() > 0
assert User.select().where(User.orcid == User.get(
email="*****@*****.**").orcid).count() == 3
assert len(User.get(email="*****@*****.**").org_links) == 3
user = User.get(email="*****@*****.**")
available_organisations = user.available_organisations
assert available_organisations.count() == 10
admin = User.create(email="*****@*****.**", organisation=user.organisation, confirmed=True,
first_name="TEST", last_name="ADMIN", roles=Role.ADMIN)
ui = UserInvitation.create(email=user.email, invitee=user, inviter=admin, token="TOKEN-123")
admin.delete_instance()
ui = UserInvitation.get(ui.id)
assert ui.inviter_id is None
user.delete_instance()
assert not UserInvitation.select().where(UserInvitation.id == ui.id).exists()
org = Organisation.select().limit(1).first()
user = User.select().limit(1).first()
ot = OrcidToken.create(user=user, org=org, scope="S1,S2,S3")
assert len(ot.scopes) == 3
ot.scopes = ["A", "B", "C", "D"]
assert ot.scope == "A,B,C,D"
def test_scheduled_tasks(app, mocker):
"""Test scheduled tasks."""
org = app.data["org"]
task = Task.create(org=org, task_type=TaskType.AFFILIATION)
rq = app.extensions["rq2"]
s = rq.get_scheduler()
s.run(burst=True)
task = Task.get(task.id)
utils.process_tasks.queue()
task = Task.get(task.id)
assert task.expires_at is not None
task.expires_at = utils.datetime(1988, 1, 1)
task.save()
utils.process_tasks.queue()
assert Task.select().where(Task.id == task.id).count() == 0
Organisation.update(webhook_enabled=True,
email_notifications_enabled=True).execute()
User.update(orcid_updated_at=utils.date.today().replace(day=1) -
utils.timedelta(days=1)).execute()
send_email = mocker.patch("orcid_hub.utils.send_email")
utils.send_orcid_update_summary.queue(org_id=org.id)
send_email.assert_called()
def test_peer_review_api(client):
"""Test peer review API in various formats."""
admin = client.data.get("admin")
resp = client.login(admin, follow_redirects=True)
resp = client.post("/load/researcher/peer_review",
data={
"file_": (open(
os.path.join(os.path.dirname(__file__), "data",
"example_peer_reviews.json"),
"rb"), "peer-reviews042.json"),
},
follow_redirects=True)
assert resp.status_code == 200
assert Task.select().count() == 1
access_token = client.get_access_token()
resp = client.get("/api/v1.0/tasks?type=PEER_REVIEW",
headers=dict(authorization=f"Bearer {access_token}"))
data = json.loads(resp.data)
task_id = int(data[0]["id"])
resp = client.get(f"/api/v1.0/peer-reviews/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
data = json.loads(resp.data)
assert len(data["records"]) == 1
assert data["filename"] == "peer-reviews042.json"
del (data["id"])
resp = client.post("/api/v1.0/peer-reviews/?filename=peer_reviews333.json",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(data))
assert resp.status_code == 200
assert Task.select().count() == 2
records = data["records"]
resp = client.post("/api/v1.0/peer-reviews/?filename=peer_reviews444.json",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(records))
assert resp.status_code == 200
assert Task.select().count() == 3
resp = client.post("/api/v1.0/peer-reviews",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(records))
assert resp.status_code == 200
assert Task.select().count() == 4
resp = client.post(f"/api/v1.0/peer-reviews/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(records))
assert resp.status_code == 200
assert Task.select().count() == 4
resp = client.head(f"/api/v1.0/peer-reviews/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert "Last-Modified" in resp.headers
assert resp.status_code == 200
resp = client.delete(f"/api/v1.0/peer-reviews/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 200
assert Task.select().count() == 3
resp = client.head(f"/api/v1.0/peer-reviews/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 404
def test_funding_api(client):
"""Test funding API in various formats."""
admin = client.data.get("admin")
resp = client.login(admin, follow_redirects=True)
resp = client.post(
"/load/researcher/funding",
data={
"file_": (
BytesIO(
"""Put Code,Title,Translated Title,Translated Title Language Code,Type,Organization Defined Type,Short Description,Amount,Currency,Start Date,End Date,Org Name,City,Region,Country,Disambiguated Org Identifier,Disambiguation Source,Visibility,ORCID iD,Email,First Name,Last Name,Name,Role,External Id Type,External Id Value,External Id Url,External Id Relationship,Identifier
,This is the project title,,,CONTRACT,Fast-Start,This is the project abstract,300000,NZD,2018,2021,Marsden Fund,Wellington,,NZ,http://dx.doi.org/10.13039/501100009193,FUNDREF,,0000-0002-9207-4933,,,,Associate Professor A Contributor 1,lead,grant_number,XXX1701,,SELF,
,This is the project title,,,CONTRACT,Fast-Start,This is the project abstract,300000,NZD,2018,2021,Marsden Fund,Wellington,,NZ,http://dx.doi.org/10.13039/501100009193,FUNDREF,,,,,,Dr B Contributor 2,co_lead,grant_number,XXX1701,,SELF,
,This is the project title,,,CONTRACT,Fast-Start,This is the project abstract,300000,NZD,2018,2021,Marsden Fund,Wellington,,NZ,http://dx.doi.org/10.13039/501100009193,FUNDREF,,,,,,Dr E Contributor 3,,grant_number,XXX1701,,SELF,
,This is the project title,,,CONTRACT,Fast-Start,This is the project abstract,300000,NZD,2018,2021,Marsden Fund,Wellington,,NZ,http://dx.doi.org/10.13039/501100009193,FUNDREF,,,[email protected],FN,LN,Dr E Contributor 4,,grant_number,XXX1701,,SELF,
,This is another project title,,,CONTRACT,Standard,This is another project abstract,800000,NZD,2018,2021,Marsden Fund,Wellington,,NZ,http://dx.doi.org/10.13039/501100009193,FUNDREF,,,[email protected],,,Associate Professor F Contributor 4,lead,grant_number,XXX1702,,SELF,9999
,This is another project title,,,CONTRACT,Standard,This is another project abstract,800000,NZD,2018,2021,Marsden Fund,Wellington,,NZ,http://dx.doi.org/10.13039/501100009193,FUNDREF,,,[email protected],John,Doe,,co_lead,grant_number,XXX1702,,SELF,8888 """
.encode() # noqa: E501
), # noqa: E501
"fundings042.csv",
),
},
follow_redirects=True)
assert resp.status_code == 200
assert Task.select().count() == 1
# TODO: factor it out
resp = client.post(
"/oauth/token",
content_type="application/x-www-form-urlencoded",
data=
b"grant_type=client_credentials&client_id=CLIENT_ID&client_secret=CLIENT_SECRET"
)
data = json.loads(resp.data)
access_token = data["access_token"]
resp = client.get("/api/v1.0/tasks?type=FUNDING",
headers=dict(authorization=f"Bearer {access_token}"))
data = json.loads(resp.data)
task_id = int(data[0]["id"])
resp = client.get(f"/api/v1.0/funds/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
data = json.loads(resp.data)
assert len(data["records"]) == 2
assert data["filename"] == "fundings042.csv"
resp = client.post("/api/v1.0/funds/?filename=fundings333.json",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(data))
assert resp.status_code == 200
assert Task.select().count() == 2
records = data["records"]
resp = client.post("/api/v1.0/funds/?filename=fundings444.json",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(records))
assert resp.status_code == 200
assert Task.select().count() == 3
resp = client.post(f"/api/v1.0/funds/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(records))
assert resp.status_code == 200
assert Task.select().count() == 3
resp = client.head(f"/api/v1.0/funds/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert "Last-Modified" in resp.headers
assert resp.status_code == 200
resp = client.delete(f"/api/v1.0/funds/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 200
assert Task.select().count() == 2
resp = client.head(f"/api/v1.0/funds/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 404
def test_affiliation_api(client, mocker):
"""Test affiliation API in various formats."""
exception = mocker.patch.object(client.application.logger, "exception")
capture_event = mocker.patch(
"sentry_sdk.transport.HttpTransport.capture_event")
resp = client.post(
"/oauth/token",
content_type="application/x-www-form-urlencoded",
data=
b"grant_type=client_credentials&client_id=TEST0-ID&client_secret=TEST0-SECRET"
)
data = json.loads(resp.data)
access_token = data["access_token"]
resp = client.post(
"/api/v1.0/affiliations/?filename=TEST42.csv",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="text/csv",
data=
b"First Name,Last Name,email,Organisation,Affiliation Type,Role,Department,Start Date,"
b"End Date,City,State,Country,Disambiguated Id,Disambiguated Source\n"
b"Researcher,Par,[email protected],Royal Org1,Staff,Programme Guide - "
b"ORCID,Research Funding,2016-09,,Wellington,SATE,NZ,,\n"
b"Roshan,Pawar,[email protected],Royal Org1,Staff,AAA,Research "
b"Funding,2016-09,,Wellington,SATE,NZ,,\n"
b"Roshan,Pawar,[email protected],Royal Org1,Student,BBB,Research "
b"Funding,2016-09,,Wellington,SATE,New Zealand,,")
data = json.loads(resp.data)
assert data["filename"] == "TEST42.csv"
assert data["task-type"] == "AFFILIATION"
assert len(data["records"]) == 3
task_id = data["id"]
resp = client.get("/api/v1.0/tasks",
headers=dict(authorization=f"Bearer {access_token}"))
tasks = json.loads(resp.data)
assert tasks[0]["id"] == task_id
resp = client.get("/api/v1.0/tasks?type=AFFILIATION",
headers=dict(authorization=f"Bearer {access_token}"))
tasks = json.loads(resp.data)
assert tasks[0]["id"] == task_id
resp = client.get("/api/v1.0/tasks?type=AFFILIATION&page=1&page_size=20",
headers=dict(authorization=f"Bearer {access_token}"))
tasks = json.loads(resp.data)
assert tasks[0]["id"] == task_id
task_copy = copy.deepcopy(data)
del (task_copy["id"])
task_copy["filename"] = "TASK-COPY.csv"
resp = client.post("/api/v1.0/affiliations/",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(task_copy))
assert Task.select().count() == 2
for r in data["records"]:
del (r["id"])
r["city"] = "TEST000"
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(data))
data = json.loads(resp.data)
assert len(resp.json["records"]) == 3
# should get a new set of records
del (data["records"][2])
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(data))
assert len(resp.json["records"]) == 2
incorrect_data = copy.deepcopy(data)
incorrect_data["records"].insert(0, {
"first-name": "TEST000 FN",
"last-name": "TEST000 LN",
})
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(incorrect_data))
assert resp.status_code == 422
assert resp.json["error"] == "Validation error."
resp = client.get(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
incorrect_data = copy.deepcopy(resp.json)
incorrect_data["records"].insert(
0, {
"email": "*****@*****.**",
"first-name": "TEST000 FN",
"last-name": "TEST000 LN",
})
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(incorrect_data))
assert resp.status_code == 422
assert resp.json["error"] == "Validation error."
resp = client.get(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
new_data = copy.deepcopy(resp.json)
new_data["records"].insert(
0, {
"email": "*****@*****.**",
"first-name": "TEST000 FN",
"last-name": "TEST000 LN",
"affiliation-type": "staff",
"city": "TEST000"
})
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(new_data))
data = json.loads(resp.data)
assert resp.status_code == 200
assert len(data["records"]) == 3
resp = client.put(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(data))
data = json.loads(resp.data)
assert resp.status_code == 200
assert len(resp.json["records"]) == 3
resp = client.get(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
new_data = copy.deepcopy(resp.json)
for i, r in enumerate(new_data["records"]):
new_data["records"][i] = {"id": r["id"], "is-active": True}
resp = client.patch(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(new_data))
assert resp.status_code == 200
assert len(resp.json["records"]) == 3
assert all(r["is-active"] for r in resp.json["records"])
assert all(r["city"] == "TEST000" for r in resp.json["records"])
resp = client.head(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert "Last-Modified" in resp.headers
resp = client.head("/api/v1.0/affiliations/999999999",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 404
resp = client.get("/api/v1.0/affiliations/999999999",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 404
resp = client.patch("/api/v1.0/affiliations/999999999",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(new_data))
assert resp.status_code == 404
with patch.object(Task, "get", side_effect=Exception("ERROR")):
resp = client.delete(
f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 400
assert resp.json == {
"error": "Unhandled exception occurred.",
"exception": "ERROR"
}
resp = client.delete(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert Task.select().count() == 1
resp = client.delete(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 404
other_user = User.get(email="*****@*****.**")
other_task = Task.create(created_by=other_user,
org=other_user.organisation,
filename="OTHER.csv",
task_type=TaskType.AFFILIATION)
resp = client.head(f"/api/v1.0/affiliations/{other_task.id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 403
resp = client.get(f"/api/v1.0/affiliations/{other_task.id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 403
resp = client.patch(f"/api/v1.0/affiliations/{other_task.id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(new_data))
assert resp.status_code == 403
resp = client.delete(f"/api/v1.0/affiliations/{other_task.id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 403
resp = client.patch(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=b'')
assert resp.status_code == 400
resp = client.post("/api/v1.0/affiliations/?filename=TEST42.csv",
headers=dict(authorization=f"Bearer {access_token}",
accept="text/yaml"),
content_type="text/yaml",
data="""task-type: AFFILIATION
filename: TEST42.yml
records:
- affiliation-type: student
city: Wellington
country: NZ
department: Research Funding
email: [email protected]
first-name: Roshan
last-name: Pawar
organisation: Royal Org1
role: BBB
start-date: 2016-09
- affiliation-type: staff
city: Wellington
country: NZ
department: Research Funding
email: [email protected]
first-name: Roshan
last-name: Pawar
organisation: Royal Org1
role: AAA
start-date: 2016-09
- affiliation-type: staff
city: Wellington
country: NZ
department: Research Funding
email: [email protected]
first-name: Researcher
is-active: false
last-name: Par
organisation: Royal Org1
role: Programme Guide - ORCID
start-date: 2016-09
""")
assert resp.json["filename"] == "TEST42.yml"
assert resp.json["task-type"] == "AFFILIATION"
assert len(resp.json["records"]) == 3
task_id = resp.json["id"]
task = Task.get(id=task_id)
assert task.affiliation_records.count() == 3
resp = client.put(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}",
accept="text/yaml"),
content_type="text/yaml",
data="""task-type: AFFILIATION
filename: TEST42.yml
records:
- affiliation-type: student
id: 9999999999
city: Wellington
country: NZ
department: Research Funding
email: [email protected]
first-name: Roshan
last-name: Pawar
organisation: Royal Org1
role: BBB
start-date: 2016-09
""")
assert resp.status_code == 400
assert resp.json["error"] == "Unhandled exception occurred."
assert "Instance matching query does not exist" in resp.json["exception"]
with patch.object(AffiliationRecord, "get",
side_effect=Exception("ERROR")):
resp = client.put(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}",
accept="text/yaml"),
content_type="text/yaml",
data="""task-type: AFFILIATION
filename: TEST42.yml
records:
- affiliation-type: student
id: 9999999999
city: Wellington
country: NZ
department: Research Funding
email: [email protected]
first-name: Roshan
last-name: Pawar
organisation: Royal Org1
role: BBB
start-date: 2016-09
""")
assert resp.status_code == 400
assert resp.json == {
"error": "Unhandled exception occurred.",
"exception": "ERROR"
}
resp = client.post(f"/api/v1.0/affiliations/?filename=TEST42.csv",
headers=dict(authorization=f"Bearer {access_token}",
accept="text/yaml"),
content_type="text/yaml",
data="""task-type: INCORRECT
filename: TEST42.yml
records:
- affiliation-type: student
id: 9999999999
""")
assert resp.status_code == 422
assert resp.json["error"] == "Validation error."
assert "INCORRECT" in resp.json["message"]
resp = client.post("/api/v1.0/affiliations/?filename=TEST_ERROR.csv",
headers=dict(authorization=f"Bearer {access_token}",
accept="text/yaml"),
content_type="text/yaml",
data="""task-type: AFFILIATION
filename: TEST_ERROR.yml
records:
- affiliation-type: student
something fishy is going here...
""")
assert resp.status_code == 415
assert resp.json[
"error"] == "Invalid request format. Only JSON, CSV, or TSV are acceptable."
assert "something fishy is going here..." in resp.json["message"]
exception.assert_called()
capture_event.assert_called()
def test_create_or_update_affiliation(patch, test_db, request_ctx):
"""Test create or update affiliation."""
org = Organisation.create(name="THE ORGANISATION",
tuakiri_name="THE ORGANISATION",
confirmed=True,
orcid_client_id="APP-5ZVH4JRQ0C27RVH5",
orcid_secret="Client Secret",
city="CITY",
country="COUNTRY",
disambiguation_org_id="ID",
disambiguation_org_source="SOURCE")
u = User.create(email="*****@*****.**",
name="TEST USER",
username="******",
roles=Role.RESEARCHER,
orcid="123",
confirmed=True,
organisation=org)
UserOrg.create(user=u, org=org)
t = Task.create(org=org,
filename="xyz.json",
created_by=u,
updated_by=u,
task_type=0)
AffiliationRecord.create(is_active=True,
task=t,
external_id="Test",
first_name="Test",
last_name="Test",
email="*****@*****.**",
orcid="123112311231",
organisation="asdasd",
affiliation_type="staff",
role="Test",
department="Test",
city="Test",
state="Test",
country="Test",
disambiguated_id="Test",
disambiguation_source="Test")
UserInvitation.create(invitee=u,
inviter=u,
org=org,
task=t,
email="*****@*****.**",
token="xyztoken")
OrcidToken.create(user=u,
org=org,
scope="/read-limited,/activities/update",
access_token="Test_token")
tasks = (Task.select(
Task, AffiliationRecord, User,
UserInvitation.id.alias("invitation_id"), OrcidToken
).where(
AffiliationRecord.processed_at.is_null(), AffiliationRecord.is_active,
((User.id.is_null(False)
& User.orcid.is_null(False) & OrcidToken.id.is_null(False)) |
((User.id.is_null() | User.orcid.is_null() | OrcidToken.id.is_null())
& UserInvitation.id.is_null() &
(AffiliationRecord.status.is_null()
| AffiliationRecord.status.contains("sent").__invert__())))).join(
AffiliationRecord,
on=(Task.id == AffiliationRecord.task_id)).join(
User,
JOIN.LEFT_OUTER,
on=((User.email == AffiliationRecord.email) |
(User.orcid == AffiliationRecord.orcid))).join(
Organisation,
JOIN.LEFT_OUTER,
on=(Organisation.id == Task.org_id)).
join(UserInvitation,
JOIN.LEFT_OUTER,
on=((UserInvitation.email == AffiliationRecord.email) &
(UserInvitation.task_id == Task.id))).join(
OrcidToken,
JOIN.LEFT_OUTER,
on=((OrcidToken.user_id == User.id) &
(OrcidToken.org_id == Organisation.id) &
(OrcidToken.scope.contains("/activities/update"))
)).limit(20))
for (task_id, org_id, user), tasks_by_user in groupby(
tasks, lambda t: (
t.id,
t.org_id,
t.affiliation_record.user,
)):
utils.create_or_update_affiliations(user=user,
org_id=org_id,
records=tasks_by_user)
affiliation_record = AffiliationRecord.get(task=t)
assert 12399 == affiliation_record.put_code
assert "12344" == affiliation_record.orcid
assert "Employment record was updated" in affiliation_record.status
def test_process_tasks(request_ctx):
"""Test expiration data setting and deletion of the exprired tasks."""
org = Organisation.get(name="TEST0")
super_user = User.get(email="*****@*****.**")
with patch("orcid_hub.utils.send_email") as send_email, request_ctx("/") as ctx:
login_user(super_user)
# flake8: noqa
task = Task.load_from_csv(
"""First name Last name email address Organisation Campus/Department City Course or Job title\tStart date End date Student/Staff\tCountry
FNA LBA [email protected] TEST1 Research Funding Wellington Programme Manager - ORCID 2016-09 Staff\tNew Zealand
""",
filename="TEST_TASK.tsv",
org=org)
Task.update(created_at=datetime(1999, 1, 1), updated_at=datetime(1999, 1, 1)).execute()
utils.process_tasks()
assert Task.select().count() == 1
assert not Task.select().where(Task.expires_at.is_null()).exists()
send_email.assert_called_once()
task = Task.select().first()
args, kwargs = send_email.call_args
assert "email/task_expiration.html" in args
assert kwargs["error_count"] == 0
hostname = ctx.request.host
assert kwargs["export_url"] == (
f"https://{hostname}/admin/affiliationrecord/export/csv/?task_id={task.id}")
assert kwargs["recipient"] == (
super_user.name,
super_user.email,
)
assert kwargs["subject"] == "Batch process task is about to expire"
assert kwargs["task"] == task
# After the second go everything should be deleted
utils.process_tasks()
assert Task.select().count() == 0
# Funding processing task:
task = Task.create(
created_at=datetime(1999, 1, 1),
org=org,
filename="FUNDING.json",
created_by=super_user,
updated_by=super_user,
task_type=TaskType.FUNDING.value)
Task.update(updated_at=datetime(1999, 1, 1)).execute()
assert Task.select().where(Task.expires_at.is_null()).count() == 1
utils.process_tasks()
assert Task.select().count() == 1
assert Task.select().where(Task.expires_at.is_null()).count() == 0
utils.process_tasks()
assert Task.select().count() == 0
args, kwargs = send_email.call_args
assert "email/task_expiration.html" in args
assert kwargs["error_count"] == 0
hostname = ctx.request.host
assert kwargs["export_url"] == (
f"https://{hostname}/admin/fundingrecord/export/csv/?task_id={task.id}")
assert kwargs["recipient"] == (
super_user.name,
super_user.email,
)
assert kwargs["subject"] == "Batch process task is about to expire"
assert kwargs["task"] == task
# Incorrect task type:
task = Task.create(
created_at=datetime(1999, 1, 1),
org=org,
filename="ERROR.err",
created_by=super_user,
updated_by=super_user,
task_type=-12345)
Task.update(updated_at=datetime(1999, 1, 1)).execute()
with pytest.raises(Exception, message="Unexpeced task type: -12345 (ERROR.err)."):
utils.process_tasks()
task.delete().execute()
# Cover case with an exterenal SP:
with patch("orcid_hub.utils.EXTERNAL_SP", "SOME.EXTERNAL.SP"):
Task.create(
created_at=datetime(1999, 1, 1),
org=org,
filename="FILE.file",
created_by=super_user,
updated_by=super_user)
Task.update(updated_at=datetime(1999, 1, 1)).execute()
assert Task.select().count() == 1
utils.process_tasks()
utils.process_tasks()
assert Task.select().count() == 0
def test_Task_count(test_models):
assert Task.select().count() == 30
def test_create_or_update_affiliation(app, mocker):
"""Test create or update affiliation."""
mocker.patch(
"orcid_api_v3.api.DevelopmentMemberAPIV30Api.update_employmentv3",
return_value=Mock(status=201, headers={'Location': '12344/XYZ/12399'}))
mocker.patch(
"orcid_api_v3.api.DevelopmentMemberAPIV30Api.create_employmentv3",
return_value=Mock(status=201, headers={'Location': '12344/XYZ/12399'}))
send_email = mocker.patch("orcid_hub.utils.send_email")
capture_event = mocker.patch(
"sentry_sdk.transport.HttpTransport.capture_event")
org = app.data["org"]
u = User.create(email="*****@*****.**",
name="TEST USER",
roles=Role.RESEARCHER,
orcid="123",
confirmed=True,
organisation=org)
UserOrg.create(user=u, org=org)
t = Task.create(org=org,
filename="xyz.json",
created_by=u,
updated_by=u,
task_type=TaskType.AFFILIATION)
OrcidToken.create(user=u,
org=org,
scopes="/read-limited,/activities/update",
access_token="Test_token")
UserInvitation.create(invitee=u,
inviter=u,
org=org,
task=t,
email="*****@*****.**",
token="xyztoken")
u = User.create(email="*****@*****.**",
name="TEST USER 2",
roles=Role.RESEARCHER,
confirmed=True,
organisation=org)
UserOrg.create(user=u, org=org)
AffiliationRecord.create(is_active=True,
task=t,
local_id="Test",
first_name="Test",
last_name="Test",
email="*****@*****.**",
orcid="123112311231",
organisation="asdasd",
affiliation_type="staff",
role="Test",
department="Test",
city="Test",
state="Test",
country="NZ",
disambiguated_id="Test",
disambiguation_source="Test")
AffiliationRecord.create(is_active=True,
task=t,
local_id="Test",
first_name="Test",
last_name="Test",
email="*****@*****.**",
orcid="123112311231",
organisation=org.name,
affiliation_type="staff",
role="Test",
department="Test",
city="Test",
state="Test",
country="NZ")
AffiliationRecord.create(is_active=True,
task=t,
local_id="Test",
first_name="Test",
last_name="Test",
email="*****@*****.**",
orcid="123112311231",
organisation="ANOTHER ORG",
affiliation_type="staff",
role="Test",
department="Test",
city="Test",
state="Test",
country="NZ",
visibility="PUBLIC")
AffiliationRecord.create(is_active=True,
task=t,
local_id="Test#2",
first_name="Test2",
last_name="Test2",
email="*****@*****.**",
organisation=org.name,
affiliation_type="staff")
tasks = (Task.select(
Task, AffiliationRecord, User,
UserInvitation.id.alias("invitation_id"), OrcidToken).join(
AffiliationRecord,
on=(Task.id == AffiliationRecord.task_id).alias("record")).join(
User,
JOIN.LEFT_OUTER,
on=((User.email == AffiliationRecord.email)
| (User.orcid == AffiliationRecord.orcid))).join(
Organisation,
JOIN.LEFT_OUTER,
on=(Organisation.id == Task.org_id)).join(
UserInvitation,
JOIN.LEFT_OUTER,
on=((UserInvitation.email
== AffiliationRecord.email)
& (UserInvitation.task_id == Task.id))).
join(OrcidToken,
JOIN.LEFT_OUTER,
on=((OrcidToken.user_id == User.id)
& (OrcidToken.org_id == Organisation.id)
& (OrcidToken.scopes.contains("/activities/update")))))
app.config["SERVER_NAME"] = "orcidhub"
for (task_id, org_id, user), tasks_by_user in groupby(
tasks, lambda t: (
t.id,
t.org_id,
t.record.user,
)):
with patch("orcid_hub.orcid_client.MemberAPIV3.get_record",
return_value=get_profile()
if user.orcid else None) as get_record:
utils.create_or_update_affiliations(user=user,
org_id=org_id,
records=tasks_by_user)
get_record.assert_any_call()
affiliation_record = AffiliationRecord.select().order_by(
AffiliationRecord.id).limit(1).first()
assert 12399 == affiliation_record.put_code
assert "12344" == affiliation_record.orcid
assert ("Employment record was updated" in affiliation_record.status
or "Employment record was created" in affiliation_record.status)
capture_event.assert_called()
send_email.assert_called_once()
def test_affiliation_api(client):
"""Test affiliation API in various formats."""
resp = client.post(
"/oauth/token",
content_type="application/x-www-form-urlencoded",
data=
b"grant_type=client_credentials&client_id=TEST0-ID&client_secret=TEST0-SECRET"
)
data = json.loads(resp.data)
access_token = data["access_token"]
resp = client.post(
"/api/v1.0/affiliations/?filename=TEST42.csv",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="text/csv",
data=
b"First Name,Last Name,email,Organisation,Affiliation Type,Role,Department,Start Date,"
b"End Date,City,State,Country,Disambiguated Id,Disambiguated Source\n"
b"Researcher,Par,[email protected],Royal Org1,Staff,Programme Guide - "
b"ORCID,Research Funding,2016-09,,Wellington,SATE,NZ,,\n"
b"Roshan,Pawar,[email protected],Royal Org1,Staff,AAA,Research "
b"Funding,2016-09,,Wellington,SATE,NZ,,\n"
b"Roshan,Pawar,[email protected],Royal Org1,Student,BBB,Research "
b"Funding,2016-09,,Wellington,SATE,New Zealand,,")
data = json.loads(resp.data)
assert data["filename"] == "TEST42.csv"
assert data["task-type"] == "AFFILIATION"
assert len(data["records"]) == 3
task_id = data["id"]
resp = client.get("/api/v1.0/tasks",
headers=dict(authorization=f"Bearer {access_token}"))
tasks = json.loads(resp.data)
assert tasks[0]["id"] == task_id
task_copy = copy.deepcopy(data)
del (task_copy["id"])
task_copy["filename"] = "TASK-COPY.csv"
resp = client.post("/api/v1.0/affiliations/",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(task_copy))
assert Task.select().count() == 2
for r in data["records"]:
del (r["id"])
r["city"] = "TEST000"
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(data))
data = json.loads(resp.data)
assert len(data["records"]) == 3
# should get a new set of records
del (data["records"][2])
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(data))
new_data = json.loads(resp.data)
assert len(new_data["records"]) == 2
incorrect_data = copy.deepcopy(data)
incorrect_data["records"].insert(0, {
"first-name": "TEST000 FN",
"last-name": "TEST000 LN",
})
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(incorrect_data))
data = json.loads(resp.data)
assert resp.status_code == 422
assert data["error"] == "Validation error."
resp = client.get(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
data = json.loads(resp.data)
incorrect_data = copy.deepcopy(data)
incorrect_data["records"].insert(
0, {
"email": "*****@*****.**",
"first-name": "TEST000 FN",
"last-name": "TEST000 LN",
})
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(incorrect_data))
data = json.loads(resp.data)
assert resp.status_code == 422
assert data["error"] == "Validation error."
resp = client.get(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
data = json.loads(resp.data)
new_data = copy.deepcopy(data)
new_data["records"].insert(
0, {
"email": "*****@*****.**",
"first-name": "TEST000 FN",
"last-name": "TEST000 LN",
"affiliation-type": "staff",
"city": "TEST000"
})
resp = client.post(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(new_data))
data = json.loads(resp.data)
assert resp.status_code == 200
assert len(data["records"]) == 3
resp = client.put(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(data))
data = json.loads(resp.data)
assert resp.status_code == 200
assert len(data["records"]) == 3
resp = client.get(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
data = json.loads(resp.data)
new_data = copy.deepcopy(data)
for i, r in enumerate(new_data["records"]):
new_data["records"][i] = {"id": r["id"], "is-active": True}
resp = client.patch(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(new_data))
data = json.loads(resp.data)
assert resp.status_code == 200
assert len(data["records"]) == 3
assert all(r["is-active"] for r in data["records"])
assert all(r["city"] == "TEST000" for r in data["records"])
resp = client.head(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert "Last-Modified" in resp.headers
resp = client.head("/api/v1.0/affiliations/999999999",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 404
resp = client.get("/api/v1.0/affiliations/999999999",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 404
resp = client.patch("/api/v1.0/affiliations/999999999",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(new_data))
assert resp.status_code == 404
resp = client.delete(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert Task.select().count() == 1
resp = client.delete(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 404
other_user = User.get(email="*****@*****.**")
other_task = Task.create(created_by=other_user,
org=other_user.organisation,
filename="OTHER.csv",
task_type=TaskType.AFFILIATION)
resp = client.head(f"/api/v1.0/affiliations/{other_task.id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 403
resp = client.get(f"/api/v1.0/affiliations/{other_task.id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 403
resp = client.patch(f"/api/v1.0/affiliations/{other_task.id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=json.dumps(new_data))
assert resp.status_code == 403
resp = client.delete(f"/api/v1.0/affiliations/{other_task.id}",
headers=dict(authorization=f"Bearer {access_token}"))
assert resp.status_code == 403
resp = client.patch(f"/api/v1.0/affiliations/{task_id}",
headers=dict(authorization=f"Bearer {access_token}"),
content_type="application/json",
data=b'')
assert resp.status_code == 400
resp = client.post("/api/v1.0/affiliations/?filename=TEST42.csv",
headers=dict(authorization=f"Bearer {access_token}",
accept="text/yaml"),
content_type="text/yaml",
data="""task-type: AFFILIATION
filename: TEST42.yml
records:
- affiliation-type: student
city: Wellington
country: NZ
department: Research Funding
email: [email protected]
first-name: Roshan
last-name: Pawar
organisation: Royal Org1
role: BBB
start-date: 2016-09
- affiliation-type: staff
city: Wellington
country: NZ
department: Research Funding
email: [email protected]
first-name: Roshan
last-name: Pawar
organisation: Royal Org1
role: AAA
start-date: 2016-09
- affiliation-type: staff
city: Wellington
country: NZ
department: Research Funding
email: [email protected]
first-name: Researcher
is-active: false
last-name: Par
organisation: Royal Org1
role: Programme Guide - ORCID
start-date: 2016-09
""")
data = json.loads(resp.data)
assert data["filename"] == "TEST42.yml"
assert data["task-type"] == "AFFILIATION"
assert len(data["records"]) == 3
task_id = data["id"]
task = Task.get(id=task_id)
assert task.affiliation_records.count() == 3