def test_process_task_from_csv_with_failures(request_ctx):
"""Test task loading and processing with failures."""
org = Organisation.get(name="TEST0")
super_user = User.get(email="*****@*****.**")
with patch("emails.html") as mock_msg, request_ctx("/") as ctx:
login_user(super_user)
# flake8: noqa
task = Task.load_from_csv(
"""First name Last name email address Organisation Campus/Department City Course or Job title Start date End date Student/Staff
FNA LBA [email protected] TEST1 Research Funding Wellington Programme Manager - ORCID 2016-09 Staff
""",
filename="TEST.tsv",
org=org)
AffiliationRecord.update(is_active=True).where(
AffiliationRecord.task_id == task.id).execute()
mock_msg().send = Mock(side_effect=Exception("FAILED TO SEND EMAIL"))
utils.process_affiliation_records(10000)
rec = AffiliationRecord.select().where(AffiliationRecord.task_id == task.id).first()
assert "FAILED TO SEND EMAIL" in rec.status
assert rec.processed_at is not None
def test_test_database(models):
"""Test of the consitency of the test database."""
assert Organisation.select().count() == 10
assert User.select().count() == 63
assert OrcidToken.select().count() == 60
assert AffiliationRecord.select().count() == 10
assert FundingRecord.select().count() == 10
assert FundingContributor.select().count() == 10
assert FundingInvitee.select().count() == 10
assert ExternalId.select().count() == 10
assert WorkRecord.select().count() == 10
assert WorkContributor.select().count() == 10
assert WorkExternalId.select().count() == 10
assert WorkInvitee.select().count() == 10
assert PeerReviewRecord.select().count() == 10
assert PeerReviewExternalId.select().count() == 10
assert PeerReviewInvitee.select().count() == 10
assert ResearcherUrlRecord.select().count() == 10
assert OtherNameRecord.select().count() == 10
assert KeywordRecord.select().count() == 10
assert Task.select().count() == 30
assert UserOrgAffiliation.select().count() == 30
assert User.get(id=43).admin_for.count() == 10
assert User.get(id=1).admin_for.count() == 0
assert User.get(id=42).admin_for.count() > 0
assert User.get(id=2).organisations.count() > 0
assert Organisation.get(id=1).admins.count() == 1
assert Organisation.get(id=5).users.count() > 0
assert Organisation.get(id=5).admins.count() > 0
assert User.select().where(User.orcid == User.get(
email="*****@*****.**").orcid).count() == 3
assert len(User.get(email="*****@*****.**").org_links) == 3
user = User.get(email="*****@*****.**")
available_organisations = user.available_organisations
assert available_organisations.count() == 10
admin = User.create(email="*****@*****.**", organisation=user.organisation, confirmed=True,
first_name="TEST", last_name="ADMIN", roles=Role.ADMIN)
ui = UserInvitation.create(email=user.email, invitee=user, inviter=admin, token="TOKEN-123")
admin.delete_instance()
ui = UserInvitation.get(ui.id)
assert ui.inviter_id is None
user.delete_instance()
assert not UserInvitation.select().where(UserInvitation.id == ui.id).exists()
org = Organisation.select().limit(1).first()
user = User.select().limit(1).first()
ot = OrcidToken.create(user=user, org=org, scope="S1,S2,S3")
assert len(ot.scopes) == 3
ot.scopes = ["A", "B", "C", "D"]
assert ot.scope == "A,B,C,D"
def test_load_task_from_csv(models):
org, _ = Organisation.get_or_create(name="TEST0")
# flake8: noqa
test = AffiliationRecord.load_from_csv(
"""First name Last name email address Organisation Campus/Department City Course or Job title Start date End date Student/Staff
FNA LBA [email protected] TEST1 Research Funding Wellington Programme Manager - ORCID 2016-09 Staff
FNA LBA [email protected] TEST1 Research Funding Wellington Programme Manager - Insights and Evaluation 2014 Staff
FNA LBA [email protected] TEST0 External Affairs Wellington Senior Evaluation Officer 2011 2014 Staff
FNA LBA [email protected] TEST0 Policy and Evaluation Wellington Evaluation Officer 2005 2011 Staff
FNA LBA [email protected] TEST0 Marsden Fund Wellington Research Assessor 2001 2004 Staff
FNB LNB [email protected] TEST1 Communications and Outreach Wellington Projects and Events Coordinator 2013 Staff
FNB LNB [email protected] TEST0 Science and Education Group Wellington School Programmes Manager 2008 2013 Staff
FNB LNB TEST_FN TEST_LN <*****@*****.**> TEST0 Science and Education Group Wellington Project Manager 2000 2004 Staff
FNB LNB [email protected] TEST0 Science and Education Group Wellington Manager Special Programmes 2004 2008 Staff
""",
filename="TEST.tsv",
org=org)
assert test.record_count == 9
assert AffiliationRecord.select().count(
) == test.record_count + 10 # The 10 value is from already inserted entries.
def models(testdb):
Organisation.insert_many((dict(name="Organisation #%d" % i,
tuakiri_name="Organisation #%d" % i,
orcid_client_id="client-%d" % i,
orcid_secret="secret-%d" % i,
confirmed=(i % 2 == 0))
for i in range(10))).execute()
User.insert_many(
(dict(name="Test User #%d" % i,
first_name="Test_%d" % i,
last_name="User_%d" % i,
email="user%d@org%d.org.nz" % (i, i * 4 % 10),
confirmed=(i % 3 != 0),
roles=Role.SUPERUSER if i % 42 == 0 else Role.ADMIN if i %
13 == 0 else Role.RESEARCHER) for i in range(60))).execute()
User.insert_many((dict(name="Test User with ORCID ID 'ABC-123' #%d" % i,
orcid="ABC-123",
first_name="Test_%d" % i,
last_name="User_%d" % i,
email="user_the_same_id_%d@org%d.org.nz" % (i, i),
confirmed=True,
organisation=(i + 1),
roles=Role.RESEARCHER)
for i in range(3))).execute()
UserOrg.insert_many(
dict(user=u.id, org=u.organisation_id)
for u in User.select().where(User.orcid == "ABC-123")).execute()
UserOrg.insert_many(
(dict(is_admin=((u + o) % 23 == 0), user=u, org=o)
for (u, o) in product(range(2, 60, 4), range(2, 10)))).execute()
UserOrg.insert_many(
(dict(is_admin=True, user=43, org=o) for o in range(1, 11))).execute()
OrcidToken.insert_many((dict(user=User.get(id=1),
org=Organisation.get(id=1),
scopes="/read-limited",
access_token="Test_%d" % i)
for i in range(60))).execute()
UserOrgAffiliation.insert_many((dict(user=User.get(id=1),
organisation=Organisation.get(id=1),
department_name="Test_%d" % i,
department_city="Test_%d" % i,
role_title="Test_%d" % i,
path="Test_%d" % i,
put_code="%d" % i)
for i in range(30))).execute()
Task.insert_many((dict(org=Organisation.get(id=1),
created_by=User.get(id=1),
updated_by=User.get(id=1),
filename="Test_%d" % i,
task_type=0) for i in range(30))).execute()
AffiliationRecord.insert_many((dict(is_active=False,
task=Task.get(id=1),
put_code=90,
external_id="Test_%d" % i,
status="Test_%d" % i,
first_name="Test_%d" % i,
last_name="Test_%d" % i,
email="Test_%d" % i,
orcid="123112311231%d" % i,
organisation="Test_%d" % i,
affiliation_type="Test_%d" % i,
role="Test_%d" % i,
department="Test_%d" % i,
city="Test_%d" % i,
state="Test_%d" % i,
country="Test_%d" % i,
disambiguated_id="Test_%d" % i,
disambiguation_source="Test_%d" % i)
for i in range(10))).execute()
PropertyRecord.insert_many((dict(type="URL",
is_active=False,
task=Task.get(id=1),
put_code=90,
status="Test_%d" % i,
first_name="Test_%d" % i,
last_name="Test_%d" % i,
email="Test_%d" % i,
orcid="123112311231%d" % i,
name="Test_%d" % i,
value="Test_%d" % i,
visibility="Test_%d" % i,
display_index=i)
for i in range(10))).execute()
PropertyRecord.insert_many((dict(type="NAME",
is_active=False,
task=Task.get(id=1),
put_code=90,
status="Test_%d" % i,
first_name="Test_%d" % i,
last_name="Test_%d" % i,
email="Test_%d" % i,
orcid="123112311231%d" % i,
value="Test_%d" % i,
visibility="Test_%d" % i,
display_index=i)
for i in range(10))).execute()
PropertyRecord.insert_many((dict(type="KEYWORD",
is_active=False,
task=Task.get(id=1),
put_code=90,
status="Test_%d" % i,
first_name="Test_%d" % i,
last_name="Test_%d" % i,
email="Test_%d" % i,
orcid="123112311231%d" % i,
value="Test_%d" % i,
visibility="Test_%d" % i,
display_index=i)
for i in range(10))).execute()
FundingRecord.insert_many(
(dict(task=Task.get(id=1),
title="Test_%d" % i,
translated_title="Test_%d" % i,
translated_title_language_code="Test_%d" % i,
type="Test_%d" % i,
organization_defined_type="Test_%d" % i,
short_description="Test_%d" % i,
amount="Test_%d" % i,
currency="Test_%d" % i,
org_name="Test_%d" % i,
city="Test_%d" % i,
region="Test_%d" % i,
country="Test_%d" % i,
disambiguated_id="Test_%d" % i,
disambiguation_source="Test_%d" % i,
is_active=False,
status="Test_%d" % i) for i in range(10))).execute()
record = FundingRecord.get()
FundingContributor.insert_many((dict(record=record,
orcid="123112311231%d" % i,
name="Test_%d" % i,
role="Test_%d" % i)
for i in range(10))).execute()
FundingInvitee.insert_many((dict(record=record,
orcid="123112311231%d" % i,
first_name="Test_%d" % i,
last_name="Test_%d" % i,
put_code=i,
status="Test_%d" % i,
identifier="%d" % i,
visibility="Test_%d" % i,
email="Test_%d" % i)
for i in range(10))).execute()
ExternalId.insert_many((dict(record=record,
type="Test_%d" % i,
value="Test_%d" % i,
url="Test_%d" % i,
relationship="Test_%d" % i)
for i in range(10))).execute()
task = Task.get()
PeerReviewRecord.insert_many(
(dict(task=task,
review_group_id="issn:1212_%d" % i,
reviewer_role="reviewer_%d" % i,
review_url="xyz_%d" % i,
review_type="REVIEW_%d" % i,
subject_external_id_type="doi_%d" % i,
subject_external_id_value="1212_%d" % i,
subject_external_id_url="url/SELF_%d" % i,
subject_external_id_relationship="SELF_%d" % i,
subject_container_name="Journal title_%d" % i,
subject_type="JOURNAL_ARTICLE_%d" % i,
subject_name_title="name_%d" % i,
subject_name_subtitle="subtitle_%d" % i,
subject_name_translated_title_lang_code="en",
subject_name_translated_title="sdsd_%d" % i,
subject_url="url_%d" % i,
convening_org_name="THE ORGANISATION_%d" % i,
convening_org_city="auckland_%d" % i,
convening_org_region="auckland_%d" % i,
convening_org_country="nz_%d" % i,
convening_org_disambiguated_identifier="123_%d" % i,
convening_org_disambiguation_source="1212_%d" % i,
is_active=False) for i in range(10))).execute()
record = PeerReviewRecord.get()
PeerReviewExternalId.insert_many((dict(record=record,
type="Test1_%d" % i,
value="Test1_%d" % i,
url="Test1_%d" % i,
relationship="Test1_%d" % i)
for i in range(10))).execute()
PeerReviewInvitee.insert_many((dict(record=record,
orcid="1231123112311%d" % i,
first_name="Test1_%d" % i,
last_name="Test1_%d" % i,
put_code=i,
status="Test1_%d" % i,
identifier="1%d" % i,
visibility="PUBLIC",
email="Test1_%d" % i)
for i in range(10))).execute()
WorkRecord.insert_many((dict(task=task,
title="Test_%d" % i,
subtitle="Test_%d" % i,
translated_title="Test_%d" % i,
translated_title_language_code="Test_%d" % i,
journal_title="Test_%d" % i,
short_description="Test_%d" % i,
citation_type="Test_%d" % i,
citation_value="Test_%d" % i,
type="Test_%d" % i,
url="Test_%d" % i,
language_code="Test_%d" % i,
country="Test_%d" % i,
is_active=False,
status="Test_%d" % i)
for i in range(10))).execute()
record = WorkRecord.get()
WorkContributor.insert_many((dict(record=record,
orcid="123112311231%d" % i,
name="Test_%d" % i,
contributor_sequence="%d" % i,
role="Test_%d" % i)
for i in range(10))).execute()
WorkExternalId.insert_many((dict(record=record,
type="Test_%d" % i,
value="Test_%d" % i,
url="Test_%d" % i,
relationship="Test_%d" % i)
for i in range(10))).execute()
WorkInvitee.insert_many((dict(record=record,
orcid="123112311231%d" % i,
first_name="Test_%d" % i,
last_name="Test_%d" % i,
put_code=i,
status="Test_%d" % i,
identifier="%d" % i,
visibility="Test_%d" % i,
email="Test_%d" % i)
for i in range(10))).execute()
yield testdb
def test_create_or_update_affiliation(patch, test_db, request_ctx):
"""Test create or update affiliation."""
org = Organisation.create(name="THE ORGANISATION",
tuakiri_name="THE ORGANISATION",
confirmed=True,
orcid_client_id="APP-5ZVH4JRQ0C27RVH5",
orcid_secret="Client Secret",
city="CITY",
country="COUNTRY",
disambiguation_org_id="ID",
disambiguation_org_source="SOURCE")
u = User.create(email="*****@*****.**",
name="TEST USER",
username="******",
roles=Role.RESEARCHER,
orcid="123",
confirmed=True,
organisation=org)
UserOrg.create(user=u, org=org)
t = Task.create(org=org,
filename="xyz.json",
created_by=u,
updated_by=u,
task_type=0)
AffiliationRecord.create(is_active=True,
task=t,
external_id="Test",
first_name="Test",
last_name="Test",
email="*****@*****.**",
orcid="123112311231",
organisation="asdasd",
affiliation_type="staff",
role="Test",
department="Test",
city="Test",
state="Test",
country="Test",
disambiguated_id="Test",
disambiguation_source="Test")
UserInvitation.create(invitee=u,
inviter=u,
org=org,
task=t,
email="*****@*****.**",
token="xyztoken")
OrcidToken.create(user=u,
org=org,
scope="/read-limited,/activities/update",
access_token="Test_token")
tasks = (Task.select(
Task, AffiliationRecord, User,
UserInvitation.id.alias("invitation_id"), OrcidToken
).where(
AffiliationRecord.processed_at.is_null(), AffiliationRecord.is_active,
((User.id.is_null(False)
& User.orcid.is_null(False) & OrcidToken.id.is_null(False)) |
((User.id.is_null() | User.orcid.is_null() | OrcidToken.id.is_null())
& UserInvitation.id.is_null() &
(AffiliationRecord.status.is_null()
| AffiliationRecord.status.contains("sent").__invert__())))).join(
AffiliationRecord,
on=(Task.id == AffiliationRecord.task_id)).join(
User,
JOIN.LEFT_OUTER,
on=((User.email == AffiliationRecord.email) |
(User.orcid == AffiliationRecord.orcid))).join(
Organisation,
JOIN.LEFT_OUTER,
on=(Organisation.id == Task.org_id)).
join(UserInvitation,
JOIN.LEFT_OUTER,
on=((UserInvitation.email == AffiliationRecord.email) &
(UserInvitation.task_id == Task.id))).join(
OrcidToken,
JOIN.LEFT_OUTER,
on=((OrcidToken.user_id == User.id) &
(OrcidToken.org_id == Organisation.id) &
(OrcidToken.scope.contains("/activities/update"))
)).limit(20))
for (task_id, org_id, user), tasks_by_user in groupby(
tasks, lambda t: (
t.id,
t.org_id,
t.affiliation_record.user,
)):
utils.create_or_update_affiliations(user=user,
org_id=org_id,
records=tasks_by_user)
affiliation_record = AffiliationRecord.get(task=t)
assert 12399 == affiliation_record.put_code
assert "12344" == affiliation_record.orcid
assert "Employment record was updated" in affiliation_record.status
def test_AffiliationRecord_count(test_models):
assert AffiliationRecord.select().count() == 10
def test_create_or_update_affiliation(app, mocker):
"""Test create or update affiliation."""
mocker.patch(
"orcid_api_v3.api.DevelopmentMemberAPIV30Api.update_employmentv3",
return_value=Mock(status=201, headers={'Location': '12344/XYZ/12399'}))
mocker.patch(
"orcid_api_v3.api.DevelopmentMemberAPIV30Api.create_employmentv3",
return_value=Mock(status=201, headers={'Location': '12344/XYZ/12399'}))
send_email = mocker.patch("orcid_hub.utils.send_email")
capture_event = mocker.patch(
"sentry_sdk.transport.HttpTransport.capture_event")
org = app.data["org"]
u = User.create(email="*****@*****.**",
name="TEST USER",
roles=Role.RESEARCHER,
orcid="123",
confirmed=True,
organisation=org)
UserOrg.create(user=u, org=org)
t = Task.create(org=org,
filename="xyz.json",
created_by=u,
updated_by=u,
task_type=TaskType.AFFILIATION)
OrcidToken.create(user=u,
org=org,
scopes="/read-limited,/activities/update",
access_token="Test_token")
UserInvitation.create(invitee=u,
inviter=u,
org=org,
task=t,
email="*****@*****.**",
token="xyztoken")
u = User.create(email="*****@*****.**",
name="TEST USER 2",
roles=Role.RESEARCHER,
confirmed=True,
organisation=org)
UserOrg.create(user=u, org=org)
AffiliationRecord.create(is_active=True,
task=t,
local_id="Test",
first_name="Test",
last_name="Test",
email="*****@*****.**",
orcid="123112311231",
organisation="asdasd",
affiliation_type="staff",
role="Test",
department="Test",
city="Test",
state="Test",
country="NZ",
disambiguated_id="Test",
disambiguation_source="Test")
AffiliationRecord.create(is_active=True,
task=t,
local_id="Test",
first_name="Test",
last_name="Test",
email="*****@*****.**",
orcid="123112311231",
organisation=org.name,
affiliation_type="staff",
role="Test",
department="Test",
city="Test",
state="Test",
country="NZ")
AffiliationRecord.create(is_active=True,
task=t,
local_id="Test",
first_name="Test",
last_name="Test",
email="*****@*****.**",
orcid="123112311231",
organisation="ANOTHER ORG",
affiliation_type="staff",
role="Test",
department="Test",
city="Test",
state="Test",
country="NZ",
visibility="PUBLIC")
AffiliationRecord.create(is_active=True,
task=t,
local_id="Test#2",
first_name="Test2",
last_name="Test2",
email="*****@*****.**",
organisation=org.name,
affiliation_type="staff")
tasks = (Task.select(
Task, AffiliationRecord, User,
UserInvitation.id.alias("invitation_id"), OrcidToken).join(
AffiliationRecord,
on=(Task.id == AffiliationRecord.task_id).alias("record")).join(
User,
JOIN.LEFT_OUTER,
on=((User.email == AffiliationRecord.email)
| (User.orcid == AffiliationRecord.orcid))).join(
Organisation,
JOIN.LEFT_OUTER,
on=(Organisation.id == Task.org_id)).join(
UserInvitation,
JOIN.LEFT_OUTER,
on=((UserInvitation.email
== AffiliationRecord.email)
& (UserInvitation.task_id == Task.id))).
join(OrcidToken,
JOIN.LEFT_OUTER,
on=((OrcidToken.user_id == User.id)
& (OrcidToken.org_id == Organisation.id)
& (OrcidToken.scopes.contains("/activities/update")))))
app.config["SERVER_NAME"] = "orcidhub"
for (task_id, org_id, user), tasks_by_user in groupby(
tasks, lambda t: (
t.id,
t.org_id,
t.record.user,
)):
with patch("orcid_hub.orcid_client.MemberAPIV3.get_record",
return_value=get_profile()
if user.orcid else None) as get_record:
utils.create_or_update_affiliations(user=user,
org_id=org_id,
records=tasks_by_user)
get_record.assert_any_call()
affiliation_record = AffiliationRecord.select().order_by(
AffiliationRecord.id).limit(1).first()
assert 12399 == affiliation_record.put_code
assert "12344" == affiliation_record.orcid
assert ("Employment record was updated" in affiliation_record.status
or "Employment record was created" in affiliation_record.status)
capture_event.assert_called()
send_email.assert_called_once()