def test_insert_claim(self):
"""
It should be able to create a series of claims
"""
r = importer.insert_claims([
{'bibcode': 'b123456789123456789',
'orcidid': '0000-0000-0000-0001',
'provenance' : 'ads test'},
{'bibcode': 'b123456789123456789',
'orcidid': '0000-0000-0000-0001',
'status' : 'updated'},
importer.create_claim(bibcode='b123456789123456789',
orcidid='0000-0000-0000-0001',
status='removed')
])
self.assertEquals(len(r), 3)
self.assertTrue(len(self.app.session.query(ClaimsLog)
.filter_by(bibcode='b123456789123456789').all()) == 3)
orcid_claim = orcid_present[c]
ads_claim = updated[c]
delta = orcid_claim[1] - ads_claim[1]
if delta.total_seconds() > app.config.get('ORCID_UPDATE_WINDOW', 60):
to_claim.append(importer.create_claim(bibcode=orcid_claim[0],
orcidid=orcidid,
provenance=self.__class__.__name__,
status='updated',
date=orcid_claim[1])
)
elif msg.get('force', False):
to_claim.append(importer.create_claim(bibcode=orcid_claim[0],
orcidid=orcidid,
provenance=self.__class__.__name__,
status='forced',
date=orcid_claim[1])
)
else:
to_claim.append(importer.create_claim(bibcode=orcid_claim[0],
orcidid=orcidid,
provenance=self.__class__.__name__,
status='unchanged',
date=orcid_claim[1]))
if len(to_claim):
json_claims = importer.insert_claims(to_claim)
for claim in json_claims:
claim['bibcode_verified'] = True
self.publish(claim)
def test_ingester_logic(self, updater_retrieve_metadata):
"""Has to be able to diff orcid profile against the
existing log in a database"""
# self.maxDiff = None
orcidid = "0000-0003-3041-2092"
httpretty.register_uri(
httpretty.GET,
self.app.config["API_ORCID_EXPORT_PROFILE"] % orcidid,
content_type="application/json",
body=open(os.path.join(self.app.config["TEST_UNIT_DIR"], "stub_data", orcidid + ".ads.json")).read(),
)
httpretty.register_uri(
httpretty.GET,
re.compile(self.app.config["API_ORCID_UPDATES_ENDPOINT"] % ".*"),
content_type="application/json",
body=open(
os.path.join(self.app.config["TEST_UNIT_DIR"], "stub_data", orcidid + ".orcid-updates.json")
).read(),
)
httpretty.register_uri(
httpretty.GET,
re.compile(self.app.config["API_SOLR_QUERY_ENDPOINT"] + ".*"),
content_type="application/json",
body=open(os.path.join(self.app.config["TEST_UNIT_DIR"], "stub_data", orcidid + ".solr.json")).read(),
)
with mock.patch("ADSOrcid.pipeline.OrcidImporter.OrcidImporter.publish") as m:
worker = OrcidImporter.OrcidImporter()
worker.check_orcid_updates()
worker.publish.assert_called_with(
{"orcidid": u"0000-0003-3041-2092", "start": "1974-11-09T22:56:52.518002+00:00"},
topic="ads.orcid.fresh-claims",
)
worker.publish.reset_mock()
worker.process_payload({"orcidid": u"0000-0003-3041-2092", "start": "1974-11-09T22:56:52.518002+00:00"})
with app.session_scope() as session:
self.assertEquals(
"2015-11-05T11:37:36.381000+00:00",
session.query(KeyValue).filter(KeyValue.key == "last.check").first().value,
)
recs = []
for x in session.query(ClaimsLog).all():
recs.append(x.toJSON())
self.assertEqual(
recs,
[
{
"status": u"#full-import",
"bibcode": u"",
"created": "2015-11-05T16:37:33.381000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 1,
},
{
"status": u"claimed",
"bibcode": u"2015arXiv150304194A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 2,
},
{
"status": u"claimed",
"bibcode": u"2015AAS...22533655A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 3,
},
{
"status": u"claimed",
"bibcode": u"2014arXiv1406.4542H",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 4,
},
{
"status": u"claimed",
"bibcode": u"2015arXiv150305881C",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"Roman Chyla",
"orcidid": u"0000-0003-3041-2092",
"id": 5,
},
{
"status": u"claimed",
"bibcode": u"2015ASPC..492..150T",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 6,
},
{
"status": u"claimed",
"bibcode": u"2015ASPC..492..208G",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 7,
},
{
"status": u"claimed",
"bibcode": u"2014AAS...22325503A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 8,
},
],
)
kv = session.query(KeyValue).filter(KeyValue.key == "last.check").first()
kv.value = ""
session.commit()
# do the same stuff again (it should not bother with new recs)
with mock.patch("ADSOrcid.pipeline.OrcidImporter.OrcidImporter.publish") as m:
worker.check_orcid_updates()
assert worker.publish.call_args[0][0]["start"] != "1974-11-09T22:56:52.518002+00:00"
worker.publish.reset_mock()
worker.process_payload({"orcidid": u"0000-0003-3041-2092"})
with app.session_scope() as session:
self.assertEquals(len(session.query(ClaimsLog).all()), 8)
new_value = parser.parse(session.query(KeyValue).filter(KeyValue.key == "last.check").first().value)
self.assertEquals(
"2015-11-05T11:37:36.381000+00:00",
session.query(KeyValue).filter(KeyValue.key == "last.check").first().value,
)
# now change the date of the #full-import (this will force the logic to re-evaluate the batch against the
# existing claims)
c = session.query(ClaimsLog).filter(ClaimsLog.status == "#full-import").first()
c.created = c.created + datetime.timedelta(microseconds=1000)
worker.process_payload({"orcidid": u"0000-0003-3041-2092"})
with app.session_scope() as session:
recs = []
for x in session.query(ClaimsLog).all():
recs.append(x.toJSON())
self.assertEqual(
recs,
[
{
"status": u"#full-import",
"bibcode": u"",
"created": "2015-11-05T16:37:33.382000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 1,
},
{
"status": u"claimed",
"bibcode": u"2015arXiv150304194A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 2,
},
{
"status": u"claimed",
"bibcode": u"2015AAS...22533655A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 3,
},
{
"status": u"claimed",
"bibcode": u"2014arXiv1406.4542H",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 4,
},
{
"status": u"claimed",
"bibcode": u"2015arXiv150305881C",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"Roman Chyla",
"orcidid": u"0000-0003-3041-2092",
"id": 5,
},
{
"status": u"claimed",
"bibcode": u"2015ASPC..492..150T",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 6,
},
{
"status": u"claimed",
"bibcode": u"2015ASPC..492..208G",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 7,
},
{
"status": u"claimed",
"bibcode": u"2014AAS...22325503A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 8,
},
{
"status": u"#full-import",
"bibcode": u"",
"created": "2015-11-05T16:37:33.381000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 9,
},
{
"status": u"unchanged",
"bibcode": u"2015arXiv150304194A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 10,
},
{
"status": u"unchanged",
"bibcode": u"2015AAS...22533655A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 11,
},
{
"status": u"unchanged",
"bibcode": u"2014arXiv1406.4542H",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 12,
},
{
"status": u"unchanged",
"bibcode": u"2015arXiv150305881C",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 13,
},
{
"status": u"unchanged",
"bibcode": u"2015ASPC..492..150T",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 14,
},
{
"status": u"unchanged",
"bibcode": u"2015ASPC..492..208G",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 15,
},
{
"status": u"unchanged",
"bibcode": u"2014AAS...22325503A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 16,
},
],
)
# now let's pretend that we have one extra claim and there was one deletion
with app.session_scope() as session:
session.query(ClaimsLog).filter(ClaimsLog.id > 8).delete() # clean up
session.query(ClaimsLog).filter_by(id=5).delete()
importer.insert_claims(
[
importer.create_claim(
bibcode="2014AAS...22325503A",
orcidid=orcidid,
status="removed",
date="2015-11-05 11:37:33.381000+00:00",
)
]
)
worker.process_payload({"orcidid": u"0000-0003-3041-2092"})
with app.session_scope() as session:
recs = []
for x in session.query(ClaimsLog).all():
recs.append(x.toJSON())
self.assertEqual(
recs,
[
{
"status": u"#full-import",
"bibcode": u"",
"created": "2015-11-05T16:37:33.382000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 1,
},
{
"status": u"claimed",
"bibcode": u"2015arXiv150304194A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 2,
},
{
"status": u"claimed",
"bibcode": u"2015AAS...22533655A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 3,
},
{
"status": u"claimed",
"bibcode": u"2014arXiv1406.4542H",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 4,
},
{
"status": u"claimed",
"bibcode": u"2015ASPC..492..150T",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 6,
},
{
"status": u"claimed",
"bibcode": u"2015ASPC..492..208G",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 7,
},
{
"status": u"claimed",
"bibcode": u"2014AAS...22325503A",
"created": "2015-09-16T10:59:01.721000+00:00",
"provenance": u"NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 8,
},
{
"status": u"removed",
"bibcode": u"2014AAS...22325503A",
"created": "2015-11-05T11:37:33.381000+00:00",
"provenance": u"None",
"orcidid": u"0000-0003-3041-2092",
"id": 9,
},
{
"status": u"#full-import",
"bibcode": u"",
"created": "2015-11-05T16:37:33.381000+00:00",
"provenance": u"OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 10,
},
{
"status": u"claimed",
"bibcode": u"2015arXiv150305881C",
"created": "2015-09-16T10:59:01.721000+00:00",
u"provenance": "Roman Chyla",
"orcidid": u"0000-0003-3041-2092",
"id": 11,
},
{
"status": u"claimed",
"bibcode": u"2014AAS...22325503A",
"created": "2015-09-16T10:59:01.721000+00:00",
u"provenance": "NASA ADS",
"orcidid": u"0000-0003-3041-2092",
"id": 12,
},
{
"status": u"unchanged",
"bibcode": u"2014arXiv1406.4542H",
"created": "2015-09-16T10:59:01.721000+00:00",
u"provenance": "OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 13,
},
{
"status": u"unchanged",
"bibcode": u"2015ASPC..492..150T",
"created": "2015-09-16T10:59:01.721000+00:00",
u"provenance": "OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 14,
},
{
"status": u"unchanged",
"bibcode": u"2015ASPC..492..208G",
"created": "2015-09-16T10:59:01.721000+00:00",
u"provenance": "OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 15,
},
{
"status": u"unchanged",
"bibcode": u"2015arXiv150304194A",
"created": "2015-09-16T10:59:01.721000+00:00",
u"provenance": "OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 16,
},
{
"status": u"unchanged",
"bibcode": u"2015AAS...22533655A",
"created": "2015-09-16T10:59:01.721000+00:00",
u"provenance": "OrcidImporter",
"orcidid": u"0000-0003-3041-2092",
"id": 17,
},
],
)
