def test_01_same_fulltext(self):
"""Check duplication detection on articles with the same fulltext URL"""
# A list of various URLs to check matching on
ftus = [
"http://examplejournal.telfor.rs/Published/Vol1No1/Vol1No1_A5.pdf",
"http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf",
"http://www.ujcem.med.sumdu.edu.ua/images/sampledata/2013/4/408_412_IV-020.pdf",
"http://www.psychologie-aktuell.com/fileadmin/download/ptam/1-2014_20140324/01_Geiser.pdf"
]
for ftu in ftus:
# make ourselves an example article
a = models.Article()
b = a.bibjson()
b.title = "Example article with a fulltext url"
b.add_url(ftu, urltype="fulltext")
a.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url"
y.add_url(ftu, urltype="fulltext")
# determine if there's a duplicate
articleService = DOAJ.articleService()
d = articleService.get_duplicate(z)
assert d is not None
assert d.bibjson().title == "Example article with a fulltext url"
def test_05_full_doi(self):
""" Test that we still detect duplicate DOIs when we have the full URI, not just the 10. """
# make ourselves a couple of example articles
a = models.Article()
b = a.bibjson()
b.title = "Example A article with a DOI"
b.add_identifier('doi', "https://doi.org/10.doi/123")
a.save(blocking=True)
# create an article which should not be caught by the duplicate detection
not_duplicate = models.Article()
not_duplicate_bibjson = not_duplicate.bibjson()
not_duplicate_bibjson.title = "Example C article with a DOI"
not_duplicate_bibjson.add_identifier('doi', "https://doi.org/10.doi/DIFFERENT")
not_duplicate.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for DOI"
y.add_identifier('doi', "http://doi.org/10.doi/123")
# determine if there's a duplicate
articleService = DOAJ.articleService()
dups = articleService.get_duplicates(z)
assert len(dups) == 1
# Check when we ask for one duplicate we get the most recent duplicate.
d = articleService.get_duplicate(z)
assert d is not None
assert d.bibjson().title == "Example A article with a DOI", d.bibjson().title
def retrieve(cls, id, account):
# is the article id valid?
ar = models.Article.pull(id)
if ar is None:
raise Api404Error()
# at this point we're happy to return the article if it's
# meant to be seen by the public
if ar.is_in_doaj():
try:
return OutgoingArticleDO.from_model(ar)
except:
raise Api500Error()
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None or account.is_anonymous:
raise Api401Error()
# Check we're allowed to retrieve this article
articleService = DOAJ.articleService()
if not articleService.is_legitimate_owner(ar, account.id):
raise Api404Error() # not found for this account
# Return the article
oa = OutgoingArticleDO.from_model(ar)
return oa
def retrieve(cls, id, account):
# is the article id valid?
ar = models.Article.pull(id)
if ar is None:
raise Api404Error()
# at this point we're happy to return the article if it's
# meant to be seen by the public
if ar.is_in_doaj():
return OutgoingArticleDO.from_model(ar)
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# Check we're allowed to retrieve this article
articleService = DOAJ.articleService()
if not articleService.is_legitimate_owner(ar, account.id):
raise Api404Error() # not found for this account
# Return the article
oa = OutgoingArticleDO.from_model(ar)
return oa
def test_07_both_duplication_criteria(self):
"""Check that an article is only reported once if it is duplicated by both DOI and fulltext URL"""
# make ourselves an example article
ftu = "http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf"
doi = "10.doi/123"
a = models.Article()
b = a.bibjson()
b.title = "Example article with a fulltext url and a DOI"
b.add_url(ftu, urltype="fulltext")
b.add_identifier('doi', doi)
a.save(blocking=True)
# create another article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url and a DOI"
y.add_url(ftu, urltype="fulltext")
y.add_identifier('doi', doi)
# determine if there's a duplicate
articleService = DOAJ.articleService()
d = articleService.get_duplicates(z)
assert len(d) == 1
print len(d)
assert d[0].bibjson().title == "Example article with a fulltext url and a DOI"
def test_07_both_duplication_criteria(self):
"""Check that an article is only reported once if it is duplicated by both DOI and fulltext URL"""
# make ourselves an example article
ftu = "http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf"
doi = "10.doi/123"
a = models.Article()
b = a.bibjson()
b.title = "Example article with a fulltext url and a DOI"
b.add_url(ftu, urltype="fulltext")
b.add_identifier('doi', doi)
a.save(blocking=True)
# create another article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url and a DOI"
y.add_url(ftu, urltype="fulltext")
y.add_identifier('doi', doi)
# determine if there's a duplicate
articleService = DOAJ.articleService()
d = articleService.get_duplicates(z)
assert len(d) == 1
print len(d)
assert d[0].bibjson(
).title == "Example article with a fulltext url and a DOI"
def test_05_full_doi(self):
""" Test that we still detect duplicate DOIs when we have the full URI, not just the 10. """
# make ourselves a couple of example articles
a = models.Article()
b = a.bibjson()
b.title = "Example A article with a DOI"
b.add_identifier('doi', "https://doi.org/10.doi/123")
a.save(blocking=True)
# create an article which should not be caught by the duplicate detection
not_duplicate = models.Article()
not_duplicate_bibjson = not_duplicate.bibjson()
not_duplicate_bibjson.title = "Example C article with a DOI"
not_duplicate_bibjson.add_identifier(
'doi', "https://doi.org/10.doi/DIFFERENT")
not_duplicate.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for DOI"
y.add_identifier('doi', "http://doi.org/10.doi/123")
# determine if there's a duplicate
articleService = DOAJ.articleService()
dups = articleService.get_duplicates(z)
assert len(dups) == 1
# Check when we ask for one duplicate we get the most recent duplicate.
d = articleService.get_duplicate(z)
assert d is not None
assert d.bibjson().title == "Example A article with a DOI", d.bibjson(
).title
def test_04_with_doi_instead(self):
"""Detect a duplicate using the DOI field."""
# make ourselves a couple of example articles
a = models.Article()
b = a.bibjson()
b.title = "Example A article with a DOI"
b.add_identifier('doi', "10.doi/123")
a.save(blocking=True)
# create an article which should not be caught by the duplicate detection
not_duplicate = models.Article()
not_duplicate_bibjson = not_duplicate.bibjson()
not_duplicate_bibjson.title = "Example C article with a DOI"
not_duplicate_bibjson.add_identifier('doi', "10.doi/DIFFERENT")
not_duplicate.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for DOI"
y.add_identifier('doi', "10.doi/123")
# determine if there's a duplicate
articleService = DOAJ.articleService()
dups = articleService.get_duplicates(z)
assert len(dups) == 1
# Check when we ask for one duplicate we get the most recent duplicate.
d = articleService.get_duplicate(z)
assert d is not None
assert d.bibjson().title == "Example A article with a DOI", d.bibjson(
).title
def test_04_with_doi_instead(self):
"""Detect a duplicate using the DOI field."""
# make ourselves a couple of example articles
a = models.Article()
b = a.bibjson()
b.title = "Example A article with a DOI"
b.add_identifier('doi', "10.doi/123")
a.save(blocking=True)
# create an article which should not be caught by the duplicate detection
not_duplicate = models.Article()
not_duplicate_bibjson = not_duplicate.bibjson()
not_duplicate_bibjson.title = "Example C article with a DOI"
not_duplicate_bibjson.add_identifier('doi', "10.doi/DIFFERENT")
not_duplicate.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for DOI"
y.add_identifier('doi', "10.doi/123")
# determine if there's a duplicate
articleService = DOAJ.articleService()
dups = articleService.get_duplicates(z)
assert len(dups) == 1
# Check when we ask for one duplicate we get the most recent duplicate.
d = articleService.get_duplicate(z)
assert d is not None
assert d.bibjson().title == "Example A article with a DOI", d.bibjson().title
def test_01_same_fulltext(self):
"""Check duplication detection on articles with the same fulltext URL"""
# A list of various URLs to check matching on
ftus = [
"http://examplejournal.telfor.rs/Published/Vol1No1/Vol1No1_A5.pdf",
"http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf",
"http://www.ujcem.med.sumdu.edu.ua/images/sampledata/2013/4/408_412_IV-020.pdf",
"http://www.psychologie-aktuell.com/fileadmin/download/ptam/1-2014_20140324/01_Geiser.pdf"
]
for ftu in ftus:
# make ourselves an example article
a = models.Article()
b = a.bibjson()
b.title = "Example article with a fulltext url"
b.add_url(ftu, urltype="fulltext")
a.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url"
y.add_url(ftu, urltype="fulltext")
# determine if there's a duplicate
articleService = DOAJ.articleService()
d = articleService.get_duplicate(z)
assert d is not None
assert d.bibjson().title == "Example article with a fulltext url"
def test_03_retrieve_multiple_conflict(self):
ftu = "http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf"
# make ourselves a couple of example articles
a = models.Article()
b = a.bibjson()
b.title = "Example A article with a fulltext url"
b.add_url(ftu, urltype="fulltext")
a.save(blocking=True)
# Wait a second to ensure the timestamps are different
time.sleep(1.01)
a2 = models.Article()
b2 = a2.bibjson()
b2.title = "Example B article with a fulltext url"
b2.add_url(ftu, urltype="fulltext")
a2.save(blocking=True)
# create an article which should not be caught by the duplicate detection
not_duplicate = models.Article()
not_duplicate_bibjson = not_duplicate.bibjson()
not_duplicate_bibjson.title = "Example C article with a fulltext url"
not_duplicate_bibjson.add_url("http://this.is/a/different/url",
urltype="fulltext")
not_duplicate.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url"
y.add_url(ftu, urltype="fulltext")
# determine that there are multiple duplicates
articleService = DOAJ.articleService()
with self.assertRaises(ArticleMergeConflict):
d = articleService.get_duplicate(z)
# get the xwalk to determine all duplicates
# sort both results and expectations here to avoid false alarm
# we don't care about the order of duplicates
expected = [a, a2]
expected.sort(key=lambda x: datetime.strptime(x.last_updated,
"%Y-%m-%dT%H:%M:%SZ"),
reverse=True)
# determine if there's a duplicate
l = articleService.get_duplicates(z)
assert isinstance(l, list), l
assert l is not None
l.sort(key=lambda x: datetime.strptime(x.last_updated,
"%Y-%m-%dT%H:%M:%SZ"),
reverse=True)
assert expected == l
def test_04_with_doi_instead(self):
"""Detect a duplicate using the DOI field."""
# make ourselves a couple of example articles
a = models.Article()
b = a.bibjson()
b.title = "Example A article with a DOI"
b.add_identifier('doi', "10.doi/123")
a.save(blocking=True)
# Wait a second to ensure the timestamps are different
time.sleep(1.01)
a2 = models.Article()
b2 = a2.bibjson()
b2.title = "Example B article with a DOI"
b2.add_identifier('doi', "10.doi/123")
a2.save(blocking=True)
# create an article which should not be caught by the duplicate detection
not_duplicate = models.Article()
not_duplicate_bibjson = not_duplicate.bibjson()
not_duplicate_bibjson.title = "Example C article with a DOI"
not_duplicate_bibjson.add_identifier('doi', "10.doi/DIFFERENT")
not_duplicate.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for DOI"
y.add_identifier('doi', "10.doi/123")
# determine if there's a duplicate
articleService = DOAJ.articleService()
dups = articleService.get_duplicates(z)
assert len(dups) == 2
# Check when we ask for one duplicate we get the most recent duplicate.
d = articleService.get_duplicate(z)
assert d is not None
assert d.bibjson().title == "Example B article with a DOI", d.bibjson(
).title
# get the xwalk to determine all duplicates
# sort both results and expectations here to avoid false alarm
# we don't care about the order of duplicates
expected = sorted([a, a2])
# determine if there's a duplicate
l = articleService.get_duplicates(z)
assert isinstance(l, list)
assert l
assert len(l) == 2
l.sort()
assert expected == l
def test_03_retrieve_latest(self):
ftu = "http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf"
# make ourselves a couple of example articles
a = models.Article()
b = a.bibjson()
b.title = "Example A article with a fulltext url"
b.add_url(ftu, urltype="fulltext")
a.save(blocking=True)
# Wait a second to ensure the timestamps are different
time.sleep(1.01)
a2 = models.Article()
b2 = a2.bibjson()
b2.title = "Example B article with a fulltext url"
b2.add_url(ftu, urltype="fulltext")
a2.save(blocking=True)
# create an article which should not be caught by the duplicate detection
not_duplicate = models.Article()
not_duplicate_bibjson = not_duplicate.bibjson()
not_duplicate_bibjson.title = "Example C article with a fulltext url"
not_duplicate_bibjson.add_url("http://this.is/a/different/url",
urltype="fulltext")
not_duplicate.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url"
y.add_url(ftu, urltype="fulltext")
# determine if there's a duplicate
articleService = DOAJ.articleService()
d = articleService.get_duplicate(z)
# Check when we ask for one duplicate we get the most recent duplicate.
assert d is not None
assert d.bibjson(
).title == "Example B article with a fulltext url", d.bibjson().title
# get the xwalk to determine all duplicates
# sort both results and expectations here to avoid false alarm
# we don't care about the order of duplicates
expected = sorted([a, a2])
# determine if there's a duplicate
l = articleService.get_duplicates(z)
assert isinstance(l, list), l
assert l is not None
l.sort()
assert expected == l
def test_03_retrieve_multiple_conflict(self):
ftu = "http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf"
# make ourselves a couple of example articles
a = models.Article()
b = a.bibjson()
b.title = "Example A article with a fulltext url"
b.add_url(ftu, urltype="fulltext")
a.save(blocking=True)
# Wait a second to ensure the timestamps are different
time.sleep(1.01)
a2 = models.Article()
b2 = a2.bibjson()
b2.title = "Example B article with a fulltext url"
b2.add_url(ftu, urltype="fulltext")
a2.save(blocking=True)
# create an article which should not be caught by the duplicate detection
not_duplicate = models.Article()
not_duplicate_bibjson = not_duplicate.bibjson()
not_duplicate_bibjson.title = "Example C article with a fulltext url"
not_duplicate_bibjson.add_url("http://this.is/a/different/url", urltype="fulltext")
not_duplicate.save(blocking=True)
# create a replacement article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url"
y.add_url(ftu, urltype="fulltext")
# determine that there are multiple duplicates
articleService = DOAJ.articleService()
with self.assertRaises(ArticleMergeConflict):
d = articleService.get_duplicate(z)
# get the xwalk to determine all duplicates
# sort both results and expectations here to avoid false alarm
# we don't care about the order of duplicates
expected = sorted([a, a2])
# determine if there's a duplicate
l = articleService.get_duplicates(z)
assert isinstance(l, list), l
assert l is not None
l.sort()
assert expected == l
def create(cls, data, account):
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# convert the data into a suitable article model
am = cls.prep_article(data)
articleService = DOAJ.articleService()
result = articleService.create_article(am, account)
# Check we are allowed to create an article for this journal
if result.get("fail", 0) == 1:
raise Api403Error()
return am
def delete(cls, id, account, dry_run=False):
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# now see if there's something for us to delete
ar = models.Article.pull(id)
if ar is None:
raise Api404Error()
# Check we're allowed to retrieve this article
articleService = DOAJ.articleService()
if not articleService.is_legitimate_owner(ar, account.id):
raise Api404Error() # not found for this account
# issue the delete (no record of the delete required)
if not dry_run:
ar.delete()
def delete(cls, id, account, dry_run=False):
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# now see if there's something for us to delete
ar = models.Article.pull(id)
if ar is None:
raise Api404Error()
# Check we're allowed to retrieve this article
articleService = DOAJ.articleService()
if not articleService.is_legitimate_owner(ar, account.id):
raise Api404Error() # not found for this account
# issue the delete (no record of the delete required)
if not dry_run:
ar.delete()
def test_02_different_fulltext(self):
"""Check that an article with different fulltext URLs is not considered a duplicate"""
# make ourselves an example article
a = models.Article()
b = a.bibjson()
b.title = "Example 2 article with a fulltext url"
b.add_url("http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf", urltype="fulltext")
a.save(blocking=True)
# create another article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url"
y.add_url("http://this.is/a/different/url", urltype="fulltext")
# determine if there's a duplicate
articleService = DOAJ.articleService()
d = articleService.get_duplicate(z)
assert d is None
def create(cls, data, account):
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# convert the data into a suitable article model
am = cls.prep_article(data)
articleService = DOAJ.articleService()
try:
result = articleService.create_article(am, account, add_journal_info=True)
except ArticleMergeConflict as e:
raise Api400Error(e.message)
# Check we are allowed to create an article for this journal
if result.get("fail", 0) == 1:
raise Api403Error()
return am
def test_02_different_fulltext(self):
"""Check that an article with different fulltext URLs is not considered a duplicate"""
# make ourselves an example article
a = models.Article()
b = a.bibjson()
b.title = "Example 2 article with a fulltext url"
b.add_url(
"http://www.sbe.deu.edu.tr/dergi/cilt15.say%C4%B12/06%20AKALIN.pdf",
urltype="fulltext")
a.save(blocking=True)
# create another article
z = models.Article()
y = z.bibjson()
y.title = "Replacement article for fulltext url"
y.add_url("http://this.is/a/different/url", urltype="fulltext")
# determine if there's a duplicate
articleService = DOAJ.articleService()
d = articleService.get_duplicate(z)
assert d is None
def create(cls, data, account):
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# convert the data into a suitable article model
am = cls.prep_article(data)
articleService = DOAJ.articleService()
try:
result = articleService.create_article(am,
account,
add_journal_info=True)
except ArticleMergeConflict as e:
raise Api400Error(e.message)
except ArticleNotAcceptable as e:
raise Api400Error("; ".join(e.errors))
# Check we are allowed to create an article for this journal
if result.get("fail", 0) == 1:
raise Api403Error()
return am
def update(cls, id, data, account):
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# now see if there's something for us to delete
ar = models.Article.pull(id)
if ar is None:
raise Api404Error()
# Check we're allowed to edit this article
articleService = DOAJ.articleService()
if not articleService.is_legitimate_owner(ar, account.id):
raise Api404Error() # not found for this account
# next thing to do is a structural validation of the replacement data, by instantiating the object
try:
ia = IncomingArticleDO(data)
except dataobj.DataStructureException as e:
raise Api400Error(e.message)
# if that works, convert it to an Article object bringing over everything outside the
# incoming article from the original article
new_ar = ia.to_article_model(ar)
# we need to ensure that any properties of the existing article that aren't allowed to change
# are copied over
new_ar.set_id(id)
new_ar.set_created(ar.created_date)
new_ar.bibjson().set_subjects(ar.bibjson().subjects())
new_ar = cls.__handle_journal_info(new_ar)
# finally save the new article, and return to the caller
new_ar.save()
return new_ar
def update(cls, id, data, account):
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# now see if there's something for us to delete
ar = models.Article.pull(id)
if ar is None:
raise Api404Error()
# Check we're allowed to edit this article
articleService = DOAJ.articleService()
if not articleService.is_legitimate_owner(ar, account.id):
raise Api404Error() # not found for this account
# next thing to do is a structural validation of the replacement data, by instantiating the object
try:
ia = IncomingArticleDO(data)
except dataobj.DataStructureException as e:
raise Api400Error(str(e))
# if that works, convert it to an Article object bringing over everything outside the
# incoming article from the original article
new_ar = ia.to_article_model(ar)
# we need to ensure that any properties of the existing article that aren't allowed to change
# are copied over
new_ar.set_id(id)
new_ar.set_created(ar.created_date)
new_ar.bibjson().set_subjects(ar.bibjson().subjects())
new_ar = cls.__handle_journal_info(new_ar)
# finally save the new article, and return to the caller
new_ar.save()
return new_ar
def create(cls, data, account):
# as long as authentication (in the layer above) has been successful, and the account exists, then
# we are good to proceed
if account is None:
raise Api401Error()
# convert the data into a suitable article model (raises Api400Error if doesn't conform to struct)
am = cls.prep_article(data)
articleService = DOAJ.articleService()
try:
result = articleService.create_article(am, account, add_journal_info=True)
except ArticleMergeConflict as e:
raise Api400Error(str(e))
except ArticleNotAcceptable as e:
raise Api400Error("; ".join(e.errors))
except exceptions.DuplicateArticleException as e:
raise Api403Error(str(e))
# Check we are allowed to create an article for this journal
if result.get("fail", 0) == 1:
raise Api403Error("It is not possible to create an article for this journal. Have you included in the upload an ISSN which is not associated with any journal in your account? ISSNs must match exactly the ISSNs against the journal record.")
return am
def run(self):
job = self.background_job
params = job.params
# Set up the files we need to run this task - a dir to place the report, and a place to write the article csv
outdir = self.get_param(params, "outdir",
"article_duplicates_" + dates.today())
job.add_audit_message("Saving reports to " + outdir)
if not os.path.exists(outdir):
os.makedirs(outdir)
# Location for our interim CSV file of articles
tmpdir = self.get_param(params, "tmpdir",
'tmp_article_duplicate_report')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
tmp_csvname = self.get_param(params, "article_csv", False)
tmp_csvpath, total = self._make_csv_dump(tmpdir, tmp_csvname)
# Initialise our reports
global_reportfile = 'duplicate_articles_global_' + dates.today(
) + '.csv'
global_reportpath = os.path.join(outdir, global_reportfile)
f = open(global_reportpath, "w", encoding="utf-8")
global_report = csv.writer(f)
header = [
"article_id", "article_created", "article_doi", "article_fulltext",
"article_owner", "article_issns", "article_in_doaj", "n_matches",
"match_type", "match_id", "match_created", "match_doi",
"match_fulltext", "match_owner", "match_issns", "match_in_doaj",
"owners_match", "titles_match", "article_title", "match_title"
]
global_report.writerow(header)
noids_reportfile = 'noids_' + dates.today() + '.csv'
noids_reportpath = os.path.join(outdir, noids_reportfile)
g = open(noids_reportpath, "w", encoding="utf-8")
noids_report = csv.writer(g)
header = [
"article_id", "article_created", "article_owner", "article_issns",
"article_in_doaj"
]
noids_report.writerow(header)
# Record the sets of duplicated articles
global_matches = []
a_count = 0
articleService = DOAJ.articleService()
# Read back in the article csv file we created earlier
with open(tmp_csvpath, 'r', encoding='utf-8') as t:
article_reader = csv.reader(t)
start = datetime.now()
estimated_finish = ""
for a in article_reader:
if a_count > 1 and a_count % 100 == 0:
n = datetime.now()
diff = (n - start).total_seconds()
expected_total = ((diff / a_count) * total)
estimated_finish = dates.format(
dates.after(start, expected_total))
a_count += 1
article = models.Article(
_source={
'id': a[0],
'created_date': a[1],
'bibjson': {
'identifier': json.loads(a[2]),
'link': json.loads(a[3]),
'title': a[4]
},
'admin': {
'in_doaj': json.loads(a[5])
}
})
# Get the global duplicates
try:
global_duplicates = articleService.discover_duplicates(
article,
results_per_match_type=10000,
include_article=False)
except exceptions.DuplicateArticleException:
# this means the article did not have any ids that could be used for deduplication
owner = self._lookup_owner(article)
noids_report.writerow([
article.id, article.created_date, owner,
','.join(article.bibjson().issns()),
article.is_in_doaj()
])
continue
dupcount = 0
if global_duplicates:
# Look up an article's owner
owner = self._lookup_owner(article)
# Deduplicate the DOI and fulltext duplicate lists
s = set([article.id] + [
d.id for d in global_duplicates.get('doi', []) +
global_duplicates.get('fulltext', [])
])
# remove article's own id from global_duplicates
dupcount = len(s) - 1
if s not in global_matches:
self._write_rows_from_duplicates(
article, owner, global_duplicates, global_report)
global_matches.append(s)
app.logger.debug('{0}/{1} {2} {3} {4} {5}'.format(
a_count, total, article.id, dupcount, len(global_matches),
estimated_finish))
job.add_audit_message(
'{0} articles processed for duplicates. {1} global duplicate sets found.'
.format(a_count, len(global_matches)))
f.close()
g.close()
# Delete the transient temporary files.
shutil.rmtree(tmpdir)
# Email the reports if that parameter has been set.
send_email = self.get_param(params, "email", False)
if send_email:
archive_name = "article_duplicates_" + dates.today()
email_archive(outdir, archive_name)
job.add_audit_message("email alert sent")
else:
job.add_audit_message("no email alert sent")
def run(self):
job = self.background_job
params = job.params
# Set up the files we need to run this task - a dir to place the report, and a place to write the article csv
outdir = self.get_param(params, "outdir", "article_duplicates_" + dates.today())
job.add_audit_message("Saving reports to " + outdir)
if not os.path.exists(outdir):
os.makedirs(outdir)
# Location for our interim CSV file of articles
tmpdir = self.get_param(params, "tmpdir", 'tmp_article_duplicate_report')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
tmp_csvname = self.get_param(params, "article_csv", False)
tmp_csvpath, total = self._make_csv_dump(tmpdir, tmp_csvname)
# Initialise our reports
global_reportfile = 'duplicate_articles_global_' + dates.today() + '.csv'
global_reportpath = os.path.join(outdir, global_reportfile)
f = codecs.open(global_reportpath, "wb", "utf-8")
global_report = UnicodeWriter(f)
header = ["article_id", "article_created", "article_doi", "article_fulltext", "article_owner", "article_issns", "article_in_doaj", "n_matches", "match_type", "match_id", "match_created", "match_doi", "match_fulltext", "match_owner", "match_issns", "match_in_doaj", "owners_match", "titles_match", "article_title", "match_title"]
global_report.writerow(header)
noids_reportfile = 'noids_' + dates.today() + '.csv'
noids_reportpath = os.path.join(outdir, noids_reportfile)
g = codecs.open(noids_reportpath, "wb", "utf-8")
noids_report = UnicodeWriter(g)
header = ["article_id", "article_created", "article_owner", "article_issns", "article_in_doaj"]
noids_report.writerow(header)
# Record the sets of duplicated articles
global_matches = []
a_count = 0
articleService = DOAJ.articleService()
# Read back in the article csv file we created earlier
with codecs.open(tmp_csvpath, 'rb', 'utf-8') as t:
article_reader = UnicodeReader(t)
start = datetime.now()
estimated_finish = ""
for a in article_reader:
if a_count > 1 and a_count % 100 == 0:
n = datetime.now()
diff = (n - start).total_seconds()
expected_total = ((diff / a_count) * total)
estimated_finish = dates.format(dates.after(start, expected_total))
a_count += 1
article = models.Article(_source={'id': a[0], 'created_date': a[1], 'bibjson': {'identifier': json.loads(a[2]), 'link': json.loads(a[3]), 'title': a[4]}, 'admin': {'in_doaj': json.loads(a[5])}})
# Get the global duplicates
try:
global_duplicates = articleService.discover_duplicates(article, owner=None, results_per_match_type=10000)
except exceptions.DuplicateArticleException:
# this means the article did not have any ids that could be used for deduplication
owner = self._lookup_owner(article)
noids_report.writerow([article.id, article.created_date, owner, ','.join(article.bibjson().issns()), article.is_in_doaj()])
continue
dupcount = 0
if global_duplicates:
# Look up an article's owner
owner = self._lookup_owner(article)
# Deduplicate the DOI and fulltext duplicate lists
s = set([article.id] + [d.id for d in global_duplicates.get('doi', []) + global_duplicates.get('fulltext', [])])
dupcount = len(s) - 1
if s not in global_matches:
self._write_rows_from_duplicates(article, owner, global_duplicates, global_report)
global_matches.append(s)
app.logger.debug('{0}/{1} {2} {3} {4} {5}'.format(a_count, total, article.id, dupcount, len(global_matches), estimated_finish))
job.add_audit_message('{0} articles processed for duplicates. {1} global duplicate sets found.'.format(a_count, len(global_matches)))
f.close()
g.close()
# Delete the transient temporary files.
shutil.rmtree(tmpdir)
# Email the reports if that parameter has been set.
send_email = self.get_param(params, "email", False)
if send_email:
archive_name = "article_duplicates_" + dates.today()
email_archive(outdir, archive_name)
job.add_audit_message("email alert sent")
else:
job.add_audit_message("no email alert sent")