def make_status_spread(cls, desired_output, period, role_map):
desired_output = deepcopy(desired_output)
header = desired_output[0]
del desired_output[0]
del header[0]
ranges = []
for h in header:
start = None
end = None
if period == "month":
startts = dates.parse(h, "%Y-%m")
year, month = divmod(startts.month + 1, 12)
if month == 0:
month = 12
year = year - 1
endts = datetime(startts.year + year, month, 1)
start = dates.format(startts)
end = dates.format(endts)
elif period == "year":
startts = dates.parse(h, "%Y")
endts = datetime(startts.year + 1, 1, 1)
start = dates.format(startts)
end = dates.format(endts)
ranges.append((start, end))
provs = []
for row in desired_output:
user = row[0]
del row[0]
role = role_map[user]
for i in range(len(row)):
count = row[i]
start, end = ranges[i]
status = None
if role == "associate_editor":
status = constants.APPLICATION_STATUS_COMPLETED
elif role == "editor":
status = constants.APPLICATION_STATUS_READY
elif role == "admin":
status = ADMIN_STATUSES[randint(0,
len(ADMIN_STATUSES) - 1)]
for j in range(count):
p = Provenance()
p.set_created(dates.random_date(start, end))
p.user = user
p.roles = [role]
p.type = "suggestion"
p.action = "status:" + status
p.resource_id = uuid.uuid4().hex
provs.append(p)
return provs
def make_status_spread(cls, desired_output, period, role_map):
desired_output = deepcopy(desired_output)
header = desired_output[0]
del desired_output[0]
del header[0]
ranges = []
for h in header:
start = None
end = None
if period == "month":
startts = dates.parse(h, "%Y-%m")
year, month = divmod(startts.month+1, 12)
if month == 0:
month = 12
year = year - 1
endts = datetime(startts.year + year, month, 1)
start = dates.format(startts)
end = dates.format(endts)
elif period == "year":
startts = dates.parse(h, "%Y")
endts = datetime(startts.year + 1, 1, 1)
start = dates.format(startts)
end = dates.format(endts)
ranges.append((start, end))
provs = []
for row in desired_output:
user = row[0]
del row[0]
role = role_map[user]
for i in range(len(row)):
count = row[i]
start, end = ranges[i]
status = None
if role == "associate_editor":
status = constants.APPLICATION_STATUS_COMPLETED
elif role == "editor":
status = constants.APPLICATION_STATUS_READY
elif role == "admin":
status = ADMIN_STATUSES[randint(0, len(ADMIN_STATUSES) - 1)]
for j in range(count):
p = Provenance()
p.set_created(dates.random_date(start, end))
p.user = user
p.roles = [role]
p.type = "suggestion"
p.action = "status:" + status
p.resource_id = uuid.uuid4().hex
provs.append(p)
return provs
def applications_inconsistencies(outfile_later, outfile_missing, conn):
with codecs.open(outfile_later, "wb", "utf-8") as f, codecs.open(outfile_missing, "wb", "utf-8") as g:
out_later = csv.writer(f)
out_later.writerow(["Application ID", "Application Last Updated", "Latest Provenance Recorded", "Difference"])
out_missing = UnicodeWriter(g)
out_missing.writerow(["Application ID", "Application Last Manual Update", "Latest Provenance Record", "ISSNs", "Title"])
counter = 0
for result in esprit.tasks.scroll(conn, "suggestion", keepalive="45m"):
counter += 1
application = Suggestion(**result)
print counter, application.id
# Part 1 - later provenance records exist
latest_prov = Provenance.get_latest_by_resource_id(application.id)
if latest_prov is not None:
lustamp = adjust_timestamp(application.last_updated_timestamp, APP_TIMEZONE_CUTOFF)
created = latest_prov.created_date
pstamp = latest_prov.created_timestamp
td = pstamp - lustamp
diff = td.total_seconds()
if diff > THRESHOLD:
out_later.writerow([application.id, application.last_updated, created, diff])
# Part 2 - missing journals
if application.application_status == constants.APPLICATION_STATUS_ACCEPTED:
missing = False
# find the matching journals by issn or by title
matching_journals = Journal.find_by_issn(application.bibjson().issns())
if len(matching_journals) == 0:
# Have another go, find by title
matching_journals = Journal.find_by_title(application.bibjson().title)
# if there are no matching journals, it is missing.
if len(matching_journals) == 0:
missing = True
else:
# if there are matching journals, find out if any of them are in the doaj. If none, then journal is still missing
those_in_doaj = len([j for j in matching_journals if j.is_in_doaj()])
if those_in_doaj == 0:
missing = True
# if the journal is missing, record it
if missing:
created = ""
if latest_prov is not None:
created = latest_prov.created_date
out_missing.writerow([application.id, application.last_manual_update, created, " ".join(application.bibjson().issns()), application.bibjson().title])
print "processed", counter, "suggestions"
def make_action_spread(cls, desired_output, action, period):
desired_output = deepcopy(desired_output)
header = desired_output[0]
del desired_output[0]
del header[0]
ranges = []
for h in header:
start = None
end = None
if period == "month":
startts = dates.parse(h, "%Y-%m")
year, month = divmod(startts.month + 1, 12)
if month == 0:
month = 12
year = year - 1
endts = datetime(startts.year + year, month, 1)
start = dates.format(startts)
end = dates.format(endts)
elif period == "year":
startts = dates.parse(h, "%Y")
endts = datetime(startts.year + 1, 1, 1)
start = dates.format(startts)
end = dates.format(endts)
ranges.append((start, end))
provs = []
for row in desired_output:
user = row[0]
del row[0]
for i in range(len(row)):
count = row[i]
start, end = ranges[i]
for j in range(count):
p = Provenance()
p.set_created(dates.random_date(start, end))
p.user = user
p.type = "suggestion"
p.action = action
p.resource_id = uuid.uuid4().hex
provs.append(p)
return provs
def make_action_spread(cls, desired_output, action, period):
desired_output = deepcopy(desired_output)
header = desired_output[0]
del desired_output[0]
del header[0]
ranges = []
for h in header:
start = None
end = None
if period == "month":
startts = dates.parse(h, "%Y-%m")
year, month = divmod(startts.month+1, 12)
if month == 0:
month = 12
year = year - 1
endts = datetime(startts.year + year, month, 1)
start = dates.format(startts)
end = dates.format(endts)
elif period == "year":
startts = dates.parse(h, "%Y")
endts = datetime(startts.year + 1, 1, 1)
start = dates.format(startts)
end = dates.format(endts)
ranges.append((start, end))
provs = []
for row in desired_output:
user = row[0]
del row[0]
for i in range(len(row)):
count = row[i]
start, end = ranges[i]
for j in range(count):
p = Provenance()
p.set_created(dates.random_date(start, end))
p.user = user
p.type = "suggestion"
p.action = action
p.resource_id = uuid.uuid4().hex
provs.append(p)
return provs
def applications_inconsistencies(outfile_later, outfile_missing, conn):
with codecs.open(outfile_later, "wb",
"utf-8") as f, codecs.open(outfile_missing, "wb",
"utf-8") as g:
out_later = csv.writer(f)
out_later.writerow([
"Application ID", "Application Last Updated",
"Latest Provenance Recorded", "Difference"
])
out_missing = UnicodeWriter(g)
out_missing.writerow([
"Application ID", "Application Last Manual Update",
"Latest Provenance Record", "ISSNs", "Title"
])
counter = 0
for result in esprit.tasks.scroll(conn, "suggestion", keepalive="45m"):
counter += 1
application = Suggestion(**result)
print counter, application.id
# Part 1 - later provenance records exist
latest_prov = Provenance.get_latest_by_resource_id(application.id)
if latest_prov is not None:
lustamp = adjust_timestamp(application.last_updated_timestamp,
APP_TIMEZONE_CUTOFF)
created = latest_prov.created_date
pstamp = latest_prov.created_timestamp
td = pstamp - lustamp
diff = td.total_seconds()
if diff > THRESHOLD:
out_later.writerow([
application.id, application.last_updated, created, diff
])
# Part 2 - missing journals
if application.application_status == constants.APPLICATION_STATUS_ACCEPTED:
missing = False
# find the matching journals by issn or by title
matching_journals = Journal.find_by_issn(
application.bibjson().issns())
if len(matching_journals) == 0:
# Have another go, find by title
matching_journals = Journal.find_by_title(
application.bibjson().title)
# if there are no matching journals, it is missing.
if len(matching_journals) == 0:
missing = True
else:
# if there are matching journals, find out if any of them are in the doaj. If none, then journal is still missing
those_in_doaj = len(
[j for j in matching_journals if j.is_in_doaj()])
if those_in_doaj == 0:
missing = True
# if the journal is missing, record it
if missing:
created = ""
if latest_prov is not None:
created = latest_prov.created_date
out_missing.writerow([
application.id, application.last_manual_update,
created, " ".join(application.bibjson().issns()),
application.bibjson().title
])
print "processed", counter, "suggestions"
def journals_applications_provenance(outfile_applications, outfile_accounts,
outfile_reapps, conn):
with codecs.open(outfile_applications, "wb", "utf-8") as f, codecs.open(
outfile_accounts, "wb",
"utf-8") as g, codecs.open(outfile_reapps, "wb", "utf-8") as h:
out_applications = csv.writer(f)
out_applications.writerow([
"Journal ID", "Journal Created", "Journal Reapplied",
"Application ID", "Application Last Updated", "Application Status",
"Published Diff", "Latest Edit Recorded",
"Latest Accepted Recorded"
])
out_accounts = csv.writer(g)
out_accounts.writerow([
"Journal ID", "Journal Created", "Journal Reapplied", "In DOAJ",
"Missing Account ID"
])
out_reapps = csv.writer(h)
out_reapps.writerow([
"Journal ID", "Journal Created", "Journal Reapplied",
"Application ID", "Application Created",
"Application Last Updated", "Application Last Manual Update",
"Application Status", "Published Diff"
])
counter = 0
for result in esprit.tasks.scroll(conn, "journal", keepalive="45m"):
counter += 1
journal = Journal(**result)
print counter, journal.id
# first figure out if there is a broken related application
issns = journal.bibjson().issns()
applications = Suggestion.find_by_issn(issns)
latest = None
for application in applications:
if latest is None:
latest = application
if application.last_updated_timestamp > latest.last_updated_timestamp:
latest = application
if latest is None:
continue
jcreated = journal.created_timestamp
reapp = journal.last_update_request
print counter, journal.id, reapp
if reapp is not None:
jcreated = datetime.strptime(reapp, "%Y-%m-%dT%H:%M:%SZ")
jcreated = adjust_timestamp(jcreated, JOURNAL_TIMEZONE_CUTOFF)
app_lustamp = adjust_timestamp(latest.last_updated_timestamp,
APP_TIMEZONE_CUTOFF)
# app_man_lustamp = latest.last_manual_update_timestamp # no need to adjust this one
app_man_lustamp = adjust_timestamp(
latest.last_manual_update_timestamp, APP_TIMEZONE_CUTOFF)
td = jcreated - app_lustamp
mtd = jcreated - app_man_lustamp
diff = td.total_seconds()
mdiff = mtd.total_seconds()
# was the journal created after the application by greater than the threshold?
if diff > THRESHOLD:
last_edit = ""
last_accept = ""
edit_query = deepcopy(PROV_QUERY)
edit_query["query"]["bool"]["must"][0]["term"][
"resource_id.exact"] = latest.id
edit_query["query"]["bool"]["must"][1]["term"][
"action.exact"] = "edit"
provs = Provenance.q2obj(q=edit_query)
if len(provs) > 0:
last_edit = provs[0].last_updated
accept_query = deepcopy(PROV_QUERY)
accept_query["query"]["bool"]["must"][0]["term"][
"resource_id.exact"] = latest.id
accept_query["query"]["bool"]["must"][1]["term"][
"action.exact"] = "status:accepted"
provs = Provenance.q2obj(q=accept_query)
if len(provs) > 0:
last_accept = provs[0].last_updated
out_applications.writerow([
journal.id, journal.created_date,
journal.last_update_request, latest.id,
latest.last_updated, latest.application_status, diff,
last_edit, last_accept
])
# was the journal (in doaj) created before the application by greater than the threshold, and is it in a state other than rejected
if mdiff < -1 * THRESHOLD and latest.application_status != constants.APPLICATION_STATUS_REJECTED and journal.is_in_doaj(
):
out_reapps.writerow([
journal.id, journal.created_date,
journal.last_update_request, latest.id,
latest.created_date, latest.last_updated,
latest.last_manual_update, latest.application_status, mdiff
])
# now figure out if the account is missing
owner = journal.owner
if owner is None:
out_accounts.writerow([
journal.id, journal.created_date,
journal.last_update_request,
str(journal.is_in_doaj()), "NO OWNER"
])
else:
acc = Account.pull(owner)
if acc is None:
out_accounts.writerow([
journal.id, journal.created_date,
journal.last_update_request,
str(journal.is_in_doaj()), owner
])
print "processed", counter, "journals"
def test_01_accept_application(self, name, application_type, account_type, manual_update, provenance, raises, result_provenance, result_manual_update):
###############################################
## set up
# create the application
application = None
if application_type == "save_fail":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
application.save = mock_save
Journal.save = mock_save
elif application_type == "with_current_journal":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
application.remove_notes()
application.add_note("unique 1", "2002-01-01T00:00:00Z")
application.add_note("duplicate", "2001-01-01T00:00:00Z")
cj = application.current_journal
journal = Journal(**JournalFixtureFactory.make_journal_source())
journal.set_id(cj)
journal.remove_notes()
journal.add_note("unique 2", "2003-01-01T00:00:00Z")
journal.add_note("duplicate", "2001-01-01T00:00:00Z")
journal.save(blocking=True)
elif application_type == "no_current_journal":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
application.remove_current_journal()
acc = None
if account_type == "not_allowed":
acc = Account(**AccountFixtureFactory.make_publisher_source())
elif account_type == "allowed":
acc = Account(**AccountFixtureFactory.make_managing_editor_source())
mu = None
if manual_update in ["true", "false"]:
mu = manual_update == "true"
prov = None
if provenance in ["true", "false"]:
prov = provenance == "true"
save = bool(randint(0,1))
###########################################################
# Execution
svc = DOAJ.applicationService()
if raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.accept_application(application, acc, mu, prov)
else:
journal = svc.accept_application(application, acc, mu, prov, save_journal=save, save_application=save)
# we need to sleep, so the index catches up
time.sleep(1)
# check a few common things
assert application.application_status == constants.APPLICATION_STATUS_ACCEPTED
assert application.current_journal is None
assert journal.current_application is None
assert application.related_journal == journal.id
related = journal.related_applications
if application_type == "with_current_journal":
assert len(related) == 3
elif application_type == "no_current_journal":
assert len(related) == 1
assert related[0].get("application_id") == application.id
assert related[0].get("date_accepted") is not None
if result_manual_update == "yes":
assert journal.last_manual_update is not None
assert journal.last_manual_update != "1970-01-01T00:00:00Z"
assert application.last_manual_update is not None
assert application.last_manual_update != "1970-01-01T00:00:00Z"
elif result_manual_update == "no":
assert journal.last_manual_update is None
assert application.last_manual_update is None
if application_type == "with_current_journal":
assert len(journal.notes) == 3
notevals = [note.get("note") for note in journal.notes]
assert "duplicate" in notevals
assert "unique 1" in notevals
assert "unique 2" in notevals
app_prov = Provenance.get_latest_by_resource_id(application.id)
if result_provenance == "yes":
assert app_prov is not None
elif result_provenance == "no":
assert app_prov is None
if save:
pass
def test_01_reject_application(self, name, application, application_status, account, prov, current_journal, note, save, raises=None):
#######################################
## set up
if save == "fail":
Suggestion.save = mock_save_fail
ap = None
journal = None
if application == "exists":
ap = Suggestion(**ApplicationFixtureFactory.make_application_source())
ap.set_application_status(application_status)
ap.set_id(ap.makeid())
ap.remove_notes()
if current_journal == "yes":
journal = Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True))
journal.set_id(journal.makeid())
journal.set_current_application(ap.id)
journal.save(blocking=True)
ap.set_current_journal(journal.id)
else:
ap.remove_current_journal()
acc = None
if account == "publisher":
acc = Account(**AccountFixtureFactory.make_publisher_source())
elif account == "admin":
acc = Account(**AccountFixtureFactory.make_managing_editor_source())
provenance = None
if prov != "none":
provenance = prov == "true"
thenote = None
if note == "yes":
thenote = "abcdefg"
########################################
## execute
svc = DOAJ.applicationService()
if raises is not None and raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.reject_application(ap, acc, provenance, note=thenote)
else:
svc.reject_application(ap, acc, provenance, note=thenote)
time.sleep(1)
#######################################
## Check
ap2 = Suggestion.pull(ap.id)
assert ap2 is not None
assert ap2.application_status == constants.APPLICATION_STATUS_REJECTED
assert ap2.current_journal is None
# check the updated and manually updated date are essentially the same (they can theoretically differ
# by a small amount just based on when they are set)
updated_spread = abs((ap2.last_updated_timestamp - ap2.last_manual_update_timestamp).total_seconds())
assert updated_spread <= 1.0
if current_journal == "yes" and journal is not None:
j2 = Journal.pull(journal.id)
assert j2 is not None
assert j2.current_application is None
assert ap2.related_journal == j2.id
if prov == "true":
pr = Provenance.get_latest_by_resource_id(ap.id)
assert pr is not None
if note == "yes":
assert len(ap2.notes) == 1
assert ap2.notes[0].get("note") == "abcdefg"
elif note == "no":
assert len(ap2.notes) == 0
def journals_applications_provenance(outfile_applications, outfile_accounts, outfile_reapps, conn):
with codecs.open(outfile_applications, "wb", "utf-8") as f, codecs.open(outfile_accounts, "wb", "utf-8") as g, codecs.open(outfile_reapps, "wb", "utf-8") as h:
out_applications = csv.writer(f)
out_applications.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "Application ID", "Application Last Updated", "Application Status", "Published Diff", "Latest Edit Recorded", "Latest Accepted Recorded"])
out_accounts = csv.writer(g)
out_accounts.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "In DOAJ", "Missing Account ID"])
out_reapps = csv.writer(h)
out_reapps.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "Application ID", "Application Created", "Application Last Updated", "Application Last Manual Update", "Application Status", "Published Diff"])
counter = 0
for result in esprit.tasks.scroll(conn, "journal", keepalive="45m"):
counter += 1
journal = Journal(**result)
print counter, journal.id
# first figure out if there is a broken related application
issns = journal.bibjson().issns()
applications = Suggestion.find_by_issn(issns)
latest = None
for application in applications:
if latest is None:
latest = application
if application.last_updated_timestamp > latest.last_updated_timestamp:
latest = application
if latest is None:
continue
jcreated = journal.created_timestamp
reapp = journal.last_update_request
print counter, journal.id, reapp
if reapp is not None:
jcreated = datetime.strptime(reapp, "%Y-%m-%dT%H:%M:%SZ")
jcreated = adjust_timestamp(jcreated, JOURNAL_TIMEZONE_CUTOFF)
app_lustamp = adjust_timestamp(latest.last_updated_timestamp, APP_TIMEZONE_CUTOFF)
# app_man_lustamp = latest.last_manual_update_timestamp # no need to adjust this one
app_man_lustamp = adjust_timestamp(latest.last_manual_update_timestamp, APP_TIMEZONE_CUTOFF)
td = jcreated - app_lustamp
mtd = jcreated - app_man_lustamp
diff = td.total_seconds()
mdiff = mtd.total_seconds()
# was the journal created after the application by greater than the threshold?
if diff > THRESHOLD:
last_edit = ""
last_accept = ""
edit_query = deepcopy(PROV_QUERY)
edit_query["query"]["bool"]["must"][0]["term"]["resource_id.exact"] = latest.id
edit_query["query"]["bool"]["must"][1]["term"]["action.exact"] = "edit"
provs = Provenance.q2obj(q=edit_query)
if len(provs) > 0:
last_edit = provs[0].last_updated
accept_query = deepcopy(PROV_QUERY)
accept_query["query"]["bool"]["must"][0]["term"]["resource_id.exact"] = latest.id
accept_query["query"]["bool"]["must"][1]["term"]["action.exact"] = "status:accepted"
provs = Provenance.q2obj(q=accept_query)
if len(provs) > 0:
last_accept = provs[0].last_updated
out_applications.writerow([journal.id, journal.created_date, journal.last_update_request, latest.id, latest.last_updated, latest.application_status, diff, last_edit, last_accept])
# was the journal (in doaj) created before the application by greater than the threshold, and is it in a state other than rejected
if mdiff < -1 * THRESHOLD and latest.application_status != constants.APPLICATION_STATUS_REJECTED and journal.is_in_doaj():
out_reapps.writerow([journal.id, journal.created_date, journal.last_update_request, latest.id, latest.created_date, latest.last_updated, latest.last_manual_update, latest.application_status, mdiff])
# now figure out if the account is missing
owner = journal.owner
if owner is None:
out_accounts.writerow([journal.id, journal.created_date, journal.last_update_request, str(journal.is_in_doaj()), "NO OWNER"])
else:
acc = Account.pull(owner)
if acc is None:
out_accounts.writerow([journal.id, journal.created_date, journal.last_update_request, str(journal.is_in_doaj()), owner])
print "processed", counter, "journals"
def test_01_reject_application(self,
name,
application,
application_status,
account,
prov,
current_journal,
note,
save,
raises=None):
#######################################
## set up
if save == "fail":
Suggestion.save = mock_save_fail
ap = None
journal = None
if application == "exists":
ap = Suggestion(
**ApplicationFixtureFactory.make_application_source())
ap.set_application_status(application_status)
ap.set_id(ap.makeid())
ap.remove_notes()
if current_journal == "yes":
journal = Journal(**JournalFixtureFactory.make_journal_source(
in_doaj=True))
journal.set_id(journal.makeid())
journal.set_current_application(ap.id)
journal.save(blocking=True)
ap.set_current_journal(journal.id)
else:
ap.remove_current_journal()
acc = None
if account == "publisher":
acc = Account(**AccountFixtureFactory.make_publisher_source())
elif account == "admin":
acc = Account(
**AccountFixtureFactory.make_managing_editor_source())
provenance = None
if prov != "none":
provenance = prov == "true"
thenote = None
if note == "yes":
thenote = "abcdefg"
########################################
## execute
svc = DOAJ.applicationService()
if raises is not None and raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.reject_application(ap, acc, provenance, note=thenote)
else:
svc.reject_application(ap, acc, provenance, note=thenote)
time.sleep(1)
#######################################
## Check
ap2 = Suggestion.pull(ap.id)
assert ap2 is not None
assert ap2.application_status == constants.APPLICATION_STATUS_REJECTED
assert ap2.current_journal is None
# check the updated and manually updated date are essentially the same (they can theoretically differ
# by a small amount just based on when they are set)
updated_spread = abs(
(ap2.last_updated_timestamp -
ap2.last_manual_update_timestamp).total_seconds())
assert updated_spread <= 1.0
if current_journal == "yes" and journal is not None:
j2 = Journal.pull(journal.id)
assert j2 is not None
assert j2.current_application is None
assert ap2.related_journal == j2.id
if prov == "true":
pr = Provenance.get_latest_by_resource_id(ap.id)
assert pr is not None
if note == "yes":
assert len(ap2.notes) == 1
assert ap2.notes[0].get("note") == "abcdefg"
elif note == "no":
assert len(ap2.notes) == 0