def migrate_suggestion(data):
if "bibjson" not in data:
return Suggestion(**data)
ap = data.get("bibjson").get("archiving_policy")
if ap is None:
return Suggestion(**data)
data["bibjson"]["archiving_policy"] = _reformat_data(ap)
return Suggestion(**data)
def applications_inconsistencies(outfile_later, outfile_missing, conn):
with codecs.open(outfile_later, "wb", "utf-8") as f, codecs.open(outfile_missing, "wb", "utf-8") as g:
out_later = csv.writer(f)
out_later.writerow(["Application ID", "Application Last Updated", "Latest Provenance Recorded", "Difference"])
out_missing = UnicodeWriter(g)
out_missing.writerow(["Application ID", "Application Last Manual Update", "Latest Provenance Record", "ISSNs", "Title"])
counter = 0
for result in esprit.tasks.scroll(conn, "suggestion", keepalive="45m"):
counter += 1
application = Suggestion(**result)
print counter, application.id
# Part 1 - later provenance records exist
latest_prov = Provenance.get_latest_by_resource_id(application.id)
if latest_prov is not None:
lustamp = adjust_timestamp(application.last_updated_timestamp, APP_TIMEZONE_CUTOFF)
created = latest_prov.created_date
pstamp = latest_prov.created_timestamp
td = pstamp - lustamp
diff = td.total_seconds()
if diff > THRESHOLD:
out_later.writerow([application.id, application.last_updated, created, diff])
# Part 2 - missing journals
if application.application_status == constants.APPLICATION_STATUS_ACCEPTED:
missing = False
# find the matching journals by issn or by title
matching_journals = Journal.find_by_issn(application.bibjson().issns())
if len(matching_journals) == 0:
# Have another go, find by title
matching_journals = Journal.find_by_title(application.bibjson().title)
# if there are no matching journals, it is missing.
if len(matching_journals) == 0:
missing = True
else:
# if there are matching journals, find out if any of them are in the doaj. If none, then journal is still missing
those_in_doaj = len([j for j in matching_journals if j.is_in_doaj()])
if those_in_doaj == 0:
missing = True
# if the journal is missing, record it
if missing:
created = ""
if latest_prov is not None:
created = latest_prov.created_date
out_missing.writerow([application.id, application.last_manual_update, created, " ".join(application.bibjson().issns()), application.bibjson().title])
print "processed", counter, "suggestions"
def test_03_view_application(self,
name,
account_type,
role,
owner,
application_type,
raises=None,
returns=None,
auth_reason=None):
# set up the objects
application = None
if application_type == "exists":
application = Suggestion(
**ApplicationFixtureFactory.make_application_source())
account = None
if account_type == "exists":
if role == "none":
account = Account(
**AccountFixtureFactory.make_publisher_source())
account.remove_role("publisher")
elif role == "publisher":
account = Account(
**AccountFixtureFactory.make_publisher_source())
elif role == "admin":
account = Account(
**AccountFixtureFactory.make_managing_editor_source())
if owner == "yes":
application.set_owner(account.id)
svc = DOAJ.authorisationService()
if raises is not None and raises != "":
exception = None
with self.assertRaises(EXCEPTIONS[raises]):
try:
svc.can_view_application(account, application)
except Exception as e:
exception = e
raise e
if raises == "AuthoriseException":
if auth_reason == "not_owner":
assert exception.reason == exception.NOT_OWNER
elif auth_reason == "wrong_role":
assert exception.reason == exception.WRONG_ROLE
elif returns is not None:
expected = returns == "true"
assert svc.can_view_application(account, application) is expected
else:
assert False, "Specify either raises or returns"
def get_suggesters():
q = {
"query" : {
"filtered" : {
"filter" : {
"bool": {
"should" : [
{"terms" : {"admin.application_status.exact" : ["pending", "answer received"]}},
{"missing" : {"field" : "admin.application_status"}}
]
}
}
}
},
"size" : 0,
"facets" : {
"suggester" : {
"terms" : {
"field" : "suggestion.suggester.email.exact",
"size" : 20000
}
}
}
}
es = Suggestion.query(q=q)
emails = [(term.get("term"), term.get("count")) for term in es.get("facets", {}).get("suggester", {}).get("terms", [])]
return emails
def get_suggestions(email):
q = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{
"term": {
"suggestion.suggester.email.exact": email
}
}],
"should": [{
"terms": {
"admin.application_status.exact":
["pending", "answer received"]
}
}, {
"missing": {
"field": "admin.application_status"
}
}]
}
}
}
}
}
return Suggestion.iterate(q)
def _sync_owner_to_application(self):
if self.current_application is None:
return
from portality.models import Suggestion
ca = Suggestion.pull(self.current_application)
if ca is not None and ca.owner != self.owner:
ca.set_owner(self.owner)
ca.save(sync_owner=False)
def _sync_owner_to_application(self):
if self.current_application is None:
return
from portality.models import Suggestion
ca = Suggestion.pull(self.current_application)
if ca is not None and ca.owner != self.owner:
ca.set_owner(self.owner)
ca.save(sync_owner=False)
def load_application_cases():
account = Account(**AccountFixtureFactory.make_publisher_source())
account.set_id(account.makeid())
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
application.makeid()
wrong_id = uuid.uuid4()
return [
param("a_id_acc_lock", application, application.id, account, True, raises=lock.Locked),
param("a_id_acc_nolock", application, application.id, account, False),
param("a_id_noacc_nolock", application, application.id, None, False),
param("a_noid_noacc_nolock", application, None, None, False, raises=exceptions.ArgumentException),
param("a_wid_noacc_nolock", application, wrong_id, None, False),
param("noa_id_noacc_nolock", None, application.id, None, False),
param("noa_noid_noacc_nolock", None, None, None, False, raises=exceptions.ArgumentException)
]
def change_status(ids, new_status):
statuses = [x[0] for x in Choices.application_status("admin") if x[0] != ""]
if new_status not in statuses:
raise Exception("Must use an allowed status: " + ", ".join(statuses))
for id in ids:
a = Suggestion.pull(id)
a.set_application_status(new_status)
a.save()
def change_status(ids, new_status):
statuses = [
x[0] for x in Choices.application_status("admin") if x[0] != ""
]
if new_status not in statuses:
raise Exception("Must use an allowed status: " + ", ".join(statuses))
for id in ids:
a = Suggestion.pull(id)
a.set_application_status(new_status)
a.save()
def traverse_applications():
a_report = {'country': [], 'currency': [], 'language': []}
statuses_incomplete = list(
set(constants.APPLICATION_STATUSES_ALL).difference({
constants.APPLICATION_STATUS_REJECTED,
constants.APPLICATION_STATUS_ACCEPTED
}))
for a in Suggestion.list_by_status(statuses_incomplete):
check_invalid_datasets(a, a_report)
return a_report
def make_application_spread(cls, desired_output, period):
desired_output = deepcopy(desired_output)
header = desired_output[0]
del desired_output[0]
del header[0]
ranges = []
for h in header:
start = None
end = None
if period == "month":
startts = dates.parse(h, "%Y-%m")
year, month = divmod(startts.month+1, 12)
if month == 0:
month = 12
year = year - 1
endts = datetime(startts.year + year, month, 1)
start = dates.format(startts)
end = dates.format(endts)
elif period == "year":
startts = dates.parse(h, "%Y")
endts = datetime(startts.year + 1, 1, 1)
start = dates.format(startts)
end = dates.format(endts)
ranges.append((start, end))
apps = []
for row in desired_output:
country = row[0]
del row[0]
for i in range(len(row)):
count = row[i]
start, end = ranges[i]
for j in range(count):
s = Suggestion()
s.set_created(dates.random_date(start, end))
s.bibjson().country = country
apps.append(s)
return apps
def create_edit_cases():
application_source = ApplicationFixtureFactory.make_application_source()
account_source = AccountFixtureFactory.make_publisher_source()
editable_application = Suggestion(**application_source)
editable_application.set_application_status(constants.APPLICATION_STATUS_UPDATE_REQUEST)
non_editable_application = Suggestion(**application_source)
non_editable_application.set_application_status(constants.APPLICATION_STATUS_READY)
owner_account = Account(**deepcopy(account_source))
owner_account.set_id(editable_application.owner)
non_owner_publisher = Account(**deepcopy(account_source))
non_publisher = Account(**deepcopy(account_source))
non_publisher.remove_role("publisher")
admin = Account(**deepcopy(account_source))
admin.add_role("admin")
return [
param("no_app_no_account", None, None, raises=exceptions.ArgumentException),
param("no_app_with_account", None, owner_account, raises=exceptions.ArgumentException),
param("app_no_account", editable_application, None, raises=exceptions.ArgumentException),
param("editable_app_owning_account", editable_application, owner_account, expected=True),
param("editable_app_nonowning_account", editable_application, non_owner_publisher, raises=exceptions.AuthoriseException),
param("editable_app_non_publisher_account", editable_application, non_publisher, raises=exceptions.AuthoriseException),
param("editable_app_admin_account", editable_application, admin, expected=True),
param("non_editable_app_owning_account", non_editable_application, owner_account, raises=exceptions.AuthoriseException),
param("non_editable_app_nonowning_account", non_editable_application, non_owner_publisher, raises=exceptions.AuthoriseException),
param("non_editable_app_non_publisher_account", non_editable_application, non_publisher, raises=exceptions.AuthoriseException),
param("non_editable_app_admin_account", non_editable_application, admin, expected=True)
]
def test_03_view_application(self, name, account_type, role, owner, application_type, raises=None, returns=None, auth_reason=None):
# set up the objects
application = None
if application_type == "exists":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
account = None
if account_type == "exists":
if role == "none":
account = Account(**AccountFixtureFactory.make_publisher_source())
account.remove_role("publisher")
elif role == "publisher":
account = Account(**AccountFixtureFactory.make_publisher_source())
elif role == "admin":
account = Account(**AccountFixtureFactory.make_managing_editor_source())
if owner == "yes":
application.set_owner(account.id)
svc = DOAJ.authorisationService()
if raises is not None and raises != "":
exception = None
with self.assertRaises(EXCEPTIONS[raises]):
try:
svc.can_view_application(account, application)
except Exception as e:
exception = e
raise e
if raises == "AuthoriseException":
if auth_reason == "not_owner":
assert exception.reason == exception.NOT_OWNER
elif auth_reason == "wrong_role":
assert exception.reason == exception.WRONG_ROLE
elif returns is not None:
expected = returns == "true"
assert svc.can_view_application(account, application) is expected
else:
assert False, "Specify either raises or returns"
def migrate_suggestions(source):
# read in the content
f = open(source)
xml = etree.parse(f)
f.close()
suggestions = xml.getroot()
print "migrating", str(len(suggestions)), "suggestion records"
for element in suggestions:
s = Suggestion()
# re-use the journal bibjson crosswalk
cb = _to_journal_bibjson(element)
s.set_bibjson(cb)
# explicitly set the open-access-ness
cb.set_open_access(_is_open_access(element))
# suggestion info
_to_suggestion(element, s)
s.save()
def get_suggestions(email):
q = {
"query" : {
"filtered" : {
"filter" : {
"bool" : {
"must" : [{"term" : {"suggestion.suggester.email.exact" : email}}],
"should" : [
{"terms" : {"admin.application_status.exact" : ["pending", "answer received"]}},
{"missing" : {"field" : "admin.application_status"}}
]
}
}
}
}
}
return Suggestion.iterate(q)
def get_suggesters():
q = {
"query": {
"filtered": {
"filter": {
"bool": {
"should": [{
"terms": {
"admin.application_status.exact":
["pending", "answer received"]
}
}, {
"missing": {
"field": "admin.application_status"
}
}]
}
}
}
},
"size": 0,
"facets": {
"suggester": {
"terms": {
"field": "suggestion.suggester.email.exact",
"size": 20000
}
}
}
}
es = Suggestion.query(q=q)
emails = [
(term.get("term"), term.get("count"))
for term in es.get("facets", {}).get("suggester", {}).get("terms", [])
]
return emails
def migrate_suggestions(source):
# read in the content
f = open(source)
xml = etree.parse(f)
f.close()
suggestions = xml.getroot()
print "migrating", str(len(suggestions)), "suggestion records"
for element in suggestions:
s = Suggestion()
# re-use the journal bibjson crosswalk
cb = _to_journal_bibjson(element)
s.set_bibjson(cb)
# explicitly set the open-access-ness
cb.set_open_access(_is_open_access(element))
# suggestion info
_to_suggestion(element, s)
s.save()
def test_02_application_2_journal(self, name, application_type,
manual_update_arg, app_key_properties,
current_journal, raises):
# set up for the test
#########################################
cj = None
has_seal = bool(randint(0, 1))
application = None
if application_type == "present":
application = Suggestion(
**ApplicationFixtureFactory.make_application_source())
application.set_id(application.makeid())
application.remove_contacts()
application.remove_editor_group()
application.remove_editor()
application.remove_owner()
application.remove_current_journal()
application.remove_notes()
if app_key_properties == "yes":
application.add_contact("Application",
"*****@*****.**")
application.set_editor_group("appeditorgroup")
application.set_editor("appeditor")
application.set_owner("appowner")
application.set_seal(has_seal)
application.add_note("Application Note")
if current_journal == "present":
journal = Journal(
**JournalFixtureFactory.make_journal_source())
journal.remove_contacts()
journal.add_contact("Journal", "*****@*****.**")
journal.set_editor_group("journaleditorgroup")
journal.set_editor("journaleditor")
journal.set_owner("journalowner")
journal.remove_current_application()
journal.remove_notes()
journal.add_note("Journal Note")
journal.save(blocking=True)
application.set_current_journal(journal.id)
cj = journal
elif current_journal == "missing":
application.set_current_journal("123456789987654321")
manual_update = None
if manual_update_arg == "true":
manual_update = True
elif manual_update_arg == "false":
manual_update = False
# execute the test
########################################
svc = DOAJ.applicationService()
if raises is not None and raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.application_2_journal(application, manual_update)
else:
journal = svc.application_2_journal(application, manual_update)
# check the result
######################################
assert journal is not None
assert isinstance(journal, Journal)
assert journal.is_in_doaj() is True
jbj = journal.bibjson().data
del jbj["active"]
assert jbj == application.bibjson().data
if current_journal == "present":
assert len(journal.related_applications) == 3
else:
assert len(journal.related_applications) == 1
related = journal.related_application_record(application.id)
assert related is not None
if manual_update_arg == "true":
assert journal.last_manual_update is not None and journal.last_manual_update != "1970-01-01T00:00:00Z"
if app_key_properties == "yes":
contacts = journal.contacts()
assert len(contacts) == 1
assert contacts[0].get("name") == "Application"
assert contacts[0].get("email") == "*****@*****.**"
assert journal.editor_group == "appeditorgroup"
assert journal.editor == "appeditor"
assert journal.owner == "appowner"
assert journal.has_seal() == has_seal
if current_journal == "present":
assert len(journal.notes) == 2
else:
assert len(journal.notes) == 1
elif app_key_properties == "no":
if current_journal == "present":
contacts = journal.contacts()
assert len(contacts) == 1
assert contacts[0].get("name") == "Journal"
assert contacts[0].get("email") == "*****@*****.**"
assert journal.editor_group == "journaleditorgroup"
assert journal.editor == "journaleditor"
assert journal.owner == "journalowner"
assert journal.has_seal() == has_seal
assert len(journal.notes) == 2
elif current_journal == "none" or current_journal == "missing":
contacts = journal.contacts()
assert len(contacts) == 0
assert journal.editor_group is None
assert journal.editor is None
assert journal.owner is None
assert journal.has_seal() == has_seal
assert len(journal.notes) == 1
if current_journal == "present":
assert cj.id == journal.id
assert cj.created_date == journal.created_date
def test_01_delete_application(self, name, application_type, account_type, current_journal, related_journal, raises):
###############################################
## set up
# create the test application (if needed), and the associated current_journal and related_journal in suitable states
application = None
cj = None
rj = None
if application_type == "found" or application_type == "locked":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
if current_journal == "none":
application.remove_current_journal()
elif current_journal == "not_found":
application.set_current_journal("123456789987654321")
elif current_journal == "found":
cj = Journal(**JournalFixtureFactory.make_journal_source())
cj.set_id(cj.makeid())
cj.save(blocking=True)
application.set_current_journal(cj.id)
elif current_journal == "locked":
cj = Journal(**JournalFixtureFactory.make_journal_source())
cj.set_id(cj.makeid())
cj.save(blocking=True)
application.set_current_journal(cj.id)
lock.lock(constants.LOCK_JOURNAL, cj.id, "otheruser")
if related_journal == "none":
application.remove_related_journal()
elif related_journal == "not_found":
application.set_related_journal("123456789987654321")
elif related_journal == "found":
rj = Journal(**JournalFixtureFactory.make_journal_source())
rj.set_id(rj.makeid())
rj.save(blocking=True)
application.set_related_journal(rj.id)
elif related_journal == "locked":
rj = Journal(**JournalFixtureFactory.make_journal_source())
rj.set_id(rj.makeid())
rj.save(blocking=True)
application.set_related_journal(rj.id)
lock.lock(constants.LOCK_JOURNAL, rj.id, "otheruser")
acc = None
if account_type != "none":
acc = Account(**AccountFixtureFactory.make_publisher_source())
if account_type == "not_permitted":
acc.remove_role("publisher")
if application_type == "locked":
thelock = lock.lock(constants.LOCK_APPLICATION, application.id, "otheruser")
# we can't explicitly block on the lock, but we can halt until we confirm it is saved
thelock.blockall([(thelock.id, thelock.last_updated)])
application_id = None
if application is not None:
if acc is not None:
application.set_owner(acc.id)
application.save(blocking=True)
application_id = application.id
elif application_type == "not_found":
application_id = u"sdjfasofwefkwflkajdfasjd"
###########################################################
# Execution
svc = DOAJ.applicationService()
if raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.delete_application(application_id, acc)
time.sleep(1)
check_locks(application, cj, rj, acc)
else:
svc.delete_application(application_id, acc)
# we need to sleep, so the index catches up
time.sleep(1)
# check that no locks remain set for this user
check_locks(application, cj, rj, acc)
# check that the application actually is gone
if application is not None:
assert Suggestion.pull(application.id) is None
# check that the current journal no longer has a reference to the application
if cj is not None:
cj = Journal.pull(cj.id)
assert cj.current_application is None
# check that the related journal has a record that the application was deleted
if rj is not None:
rj = Journal.pull(rj.id)
record = rj.related_application_record(application.id)
assert "status" in record
assert record["status"] == "deleted"
with open("application_2_journal.csv", "wb") as f:
writer = csv.writer(f)
writer.writerow([
"application", "app_created", "app_last_update",
"app_last_manual_update", "app_adjusted_lmu", "app_issns",
"journal_matches", "journal", "journal_created", "journal_reapp",
"journal_issns", "jc_ac_diff", "jc_lmua_diff", "mc1", "lra_ac_diff",
"mc2", "is_match", "reason"
])
# first, get each application and consider it
counter = 0
for result in esprit.tasks.scroll(conn, "suggestion", keepalive="1m"):
counter += 1
application = Suggestion(**result)
application.remove_related_journal()
# find all the journals that this application could be associated with (which we need to do by issn)
issns = application.bibjson().issns()
# query by each issn individually, because we're looking for the widest possible map. Querying by
# both would require both issns match
related_journals = []
related_journal_ids = []
for issn in issns:
journals = Journal.find_by_issn(issn)
for journal in journals:
if journal.id not in related_journal_ids:
related_journal_ids.append(journal.id)
related_journals.append(journal)
def test_01_accept_application(self, name, application_type, account_type, manual_update, provenance, raises, result_provenance, result_manual_update):
###############################################
## set up
# create the application
application = None
if application_type == "save_fail":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
application.save = mock_save
Journal.save = mock_save
elif application_type == "with_current_journal":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
application.remove_notes()
application.add_note("unique 1", "2002-01-01T00:00:00Z")
application.add_note("duplicate", "2001-01-01T00:00:00Z")
cj = application.current_journal
journal = Journal(**JournalFixtureFactory.make_journal_source())
journal.set_id(cj)
journal.remove_notes()
journal.add_note("unique 2", "2003-01-01T00:00:00Z")
journal.add_note("duplicate", "2001-01-01T00:00:00Z")
journal.save(blocking=True)
elif application_type == "no_current_journal":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
application.remove_current_journal()
acc = None
if account_type == "not_allowed":
acc = Account(**AccountFixtureFactory.make_publisher_source())
elif account_type == "allowed":
acc = Account(**AccountFixtureFactory.make_managing_editor_source())
mu = None
if manual_update in ["true", "false"]:
mu = manual_update == "true"
prov = None
if provenance in ["true", "false"]:
prov = provenance == "true"
save = bool(randint(0,1))
###########################################################
# Execution
svc = DOAJ.applicationService()
if raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.accept_application(application, acc, mu, prov)
else:
journal = svc.accept_application(application, acc, mu, prov, save_journal=save, save_application=save)
# we need to sleep, so the index catches up
time.sleep(1)
# check a few common things
assert application.application_status == constants.APPLICATION_STATUS_ACCEPTED
assert application.current_journal is None
assert journal.current_application is None
assert application.related_journal == journal.id
related = journal.related_applications
if application_type == "with_current_journal":
assert len(related) == 3
elif application_type == "no_current_journal":
assert len(related) == 1
assert related[0].get("application_id") == application.id
assert related[0].get("date_accepted") is not None
if result_manual_update == "yes":
assert journal.last_manual_update is not None
assert journal.last_manual_update != "1970-01-01T00:00:00Z"
assert application.last_manual_update is not None
assert application.last_manual_update != "1970-01-01T00:00:00Z"
elif result_manual_update == "no":
assert journal.last_manual_update is None
assert application.last_manual_update is None
if application_type == "with_current_journal":
assert len(journal.notes) == 3
notevals = [note.get("note") for note in journal.notes]
assert "duplicate" in notevals
assert "unique 1" in notevals
assert "unique 2" in notevals
app_prov = Provenance.get_latest_by_resource_id(application.id)
if result_provenance == "yes":
assert app_prov is not None
elif result_provenance == "no":
assert app_prov is None
if save:
pass
def test_01_reject_application(self, name, application, application_status, account, prov, current_journal, note, save, raises=None):
#######################################
## set up
if save == "fail":
Suggestion.save = mock_save_fail
ap = None
journal = None
if application == "exists":
ap = Suggestion(**ApplicationFixtureFactory.make_application_source())
ap.set_application_status(application_status)
ap.set_id(ap.makeid())
ap.remove_notes()
if current_journal == "yes":
journal = Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True))
journal.set_id(journal.makeid())
journal.set_current_application(ap.id)
journal.save(blocking=True)
ap.set_current_journal(journal.id)
else:
ap.remove_current_journal()
acc = None
if account == "publisher":
acc = Account(**AccountFixtureFactory.make_publisher_source())
elif account == "admin":
acc = Account(**AccountFixtureFactory.make_managing_editor_source())
provenance = None
if prov != "none":
provenance = prov == "true"
thenote = None
if note == "yes":
thenote = "abcdefg"
########################################
## execute
svc = DOAJ.applicationService()
if raises is not None and raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.reject_application(ap, acc, provenance, note=thenote)
else:
svc.reject_application(ap, acc, provenance, note=thenote)
time.sleep(1)
#######################################
## Check
ap2 = Suggestion.pull(ap.id)
assert ap2 is not None
assert ap2.application_status == constants.APPLICATION_STATUS_REJECTED
assert ap2.current_journal is None
# check the updated and manually updated date are essentially the same (they can theoretically differ
# by a small amount just based on when they are set)
updated_spread = abs((ap2.last_updated_timestamp - ap2.last_manual_update_timestamp).total_seconds())
assert updated_spread <= 1.0
if current_journal == "yes" and journal is not None:
j2 = Journal.pull(journal.id)
assert j2 is not None
assert j2.current_application is None
assert ap2.related_journal == j2.id
if prov == "true":
pr = Provenance.get_latest_by_resource_id(ap.id)
assert pr is not None
if note == "yes":
assert len(ap2.notes) == 1
assert ap2.notes[0].get("note") == "abcdefg"
elif note == "no":
assert len(ap2.notes) == 0
def test_01_update_request(self, name, journal_id, journal_lock, account,
account_role, account_is_owner,
current_applications, application_lock,
application_status, completed_applications,
raises, return_app, return_jlock, return_alock,
db_jlock, db_alock, db_app):
###############################################
## set up
# create the journal
journal = None
jid = None
if journal_id == "valid":
journal = Journal(**JournalFixtureFactory.make_journal_source(
in_doaj=True))
journal.remove_related_applications()
journal.remove_current_application()
jid = journal.id
elif journal_id == "not_in_doaj":
journal = Journal(**JournalFixtureFactory.make_journal_source(
in_doaj=False))
journal.remove_related_applications()
journal.remove_current_application()
jid = journal.id
elif journal_id == "missing":
jid = uuid.uuid4().hex
acc = None
if account == "yes":
acc = Account(**AccountFixtureFactory.make_publisher_source())
if account_role == "none":
acc.remove_role("publisher")
elif account_role == "admin":
acc.remove_role("publisher")
acc.add_role("admin")
acc.set_id(acc.makeid())
if account_is_owner == "yes":
acc.set_id(journal.owner)
if journal_lock == "yes":
lock.lock("journal", jid, "someoneelse", blocking=True)
latest_app = None
current_app_count = int(current_applications)
for i in range(current_app_count):
app = Suggestion(
**ApplicationFixtureFactory.make_application_source())
app.set_id(app.makeid())
app.set_created("198" + str(i) + "-01-01T00:00:00Z")
app.set_current_journal(jid)
app.save()
latest_app = app
if journal is not None:
journal.set_current_application(app.id)
comp_app_count = int(completed_applications)
for i in range(comp_app_count):
app = Suggestion(
**ApplicationFixtureFactory.make_application_source())
app.set_id(app.makeid())
app.set_created("197" + str(i) + "-01-01T00:00:00Z")
app.set_related_journal(jid)
app.save()
if journal is not None:
journal.add_related_application(app.id,
date_accepted=app.created_date)
if current_app_count == 0 and comp_app_count == 0:
# save at least one record to initialise the index mapping, otherwise tests fail
app = Suggestion(
**ApplicationFixtureFactory.make_application_source())
app.set_id(app.makeid())
app.save()
if application_lock == "yes":
lock.lock("suggestion",
latest_app.id,
"someoneelse",
blocking=True)
if application_status != "n/a":
latest_app.set_application_status(application_status)
latest_app.save(blocking=True)
# finally save the journal record, ensuring we get a blocking save, so everything
# above here should be synchronised with the repo
if journal is not None:
journal.save(blocking=True)
###########################################################
# Execution
svc = DOAJ.applicationService()
if raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.update_request_for_journal(jid, acc)
else:
application, jlock, alock = svc.update_request_for_journal(
jid, acc)
# we need to sleep, so the index catches up
time.sleep(1)
if return_app == "none":
assert application is None
elif return_app == "yes":
assert application is not None
if return_jlock == "none":
assert jlock is None
elif return_jlock == "yes":
assert jlock is not None
if return_alock == "none":
assert alock is None
elif return_alock == "yes":
assert alock is not None
if db_jlock == "no" and acc is not None:
assert not lock.has_lock("journal", jid, acc.id)
elif db_jlock == "yes" and acc is not None:
l = lock.has_lock("journal", jid, acc.id)
assert lock.has_lock("journal", jid, acc.id)
if db_alock == "no" and application.id is not None and acc is not None:
assert not lock.has_lock("suggestion", application.id, acc.id)
elif db_alock == "yes" and application.id is not None and acc is not None:
assert lock.has_lock("suggestion", application.id, acc.id)
if db_app == "no" and application.id is not None:
indb = Suggestion.q2obj(q="id.exact:" + application.id)
assert indb is None
elif db_app == "yes" and application.id is not None:
indb = Suggestion.q2obj(q="id.exact:" + application.id)
assert indb is not None
if current_app_count == 0 and comp_app_count == 0 and application is not None:
assert application.article_metadata is None
assert application.articles_last_year is None
elif application is not None:
assert application.article_metadata is not None
assert application.articles_last_year is not None
def journals_applications_provenance(outfile_applications, outfile_accounts,
outfile_reapps, conn):
with codecs.open(outfile_applications, "wb", "utf-8") as f, codecs.open(
outfile_accounts, "wb",
"utf-8") as g, codecs.open(outfile_reapps, "wb", "utf-8") as h:
out_applications = csv.writer(f)
out_applications.writerow([
"Journal ID", "Journal Created", "Journal Reapplied",
"Application ID", "Application Last Updated", "Application Status",
"Published Diff", "Latest Edit Recorded",
"Latest Accepted Recorded"
])
out_accounts = csv.writer(g)
out_accounts.writerow([
"Journal ID", "Journal Created", "Journal Reapplied", "In DOAJ",
"Missing Account ID"
])
out_reapps = csv.writer(h)
out_reapps.writerow([
"Journal ID", "Journal Created", "Journal Reapplied",
"Application ID", "Application Created",
"Application Last Updated", "Application Last Manual Update",
"Application Status", "Published Diff"
])
counter = 0
for result in esprit.tasks.scroll(conn, "journal", keepalive="45m"):
counter += 1
journal = Journal(**result)
print counter, journal.id
# first figure out if there is a broken related application
issns = journal.bibjson().issns()
applications = Suggestion.find_by_issn(issns)
latest = None
for application in applications:
if latest is None:
latest = application
if application.last_updated_timestamp > latest.last_updated_timestamp:
latest = application
if latest is None:
continue
jcreated = journal.created_timestamp
reapp = journal.last_update_request
print counter, journal.id, reapp
if reapp is not None:
jcreated = datetime.strptime(reapp, "%Y-%m-%dT%H:%M:%SZ")
jcreated = adjust_timestamp(jcreated, JOURNAL_TIMEZONE_CUTOFF)
app_lustamp = adjust_timestamp(latest.last_updated_timestamp,
APP_TIMEZONE_CUTOFF)
# app_man_lustamp = latest.last_manual_update_timestamp # no need to adjust this one
app_man_lustamp = adjust_timestamp(
latest.last_manual_update_timestamp, APP_TIMEZONE_CUTOFF)
td = jcreated - app_lustamp
mtd = jcreated - app_man_lustamp
diff = td.total_seconds()
mdiff = mtd.total_seconds()
# was the journal created after the application by greater than the threshold?
if diff > THRESHOLD:
last_edit = ""
last_accept = ""
edit_query = deepcopy(PROV_QUERY)
edit_query["query"]["bool"]["must"][0]["term"][
"resource_id.exact"] = latest.id
edit_query["query"]["bool"]["must"][1]["term"][
"action.exact"] = "edit"
provs = Provenance.q2obj(q=edit_query)
if len(provs) > 0:
last_edit = provs[0].last_updated
accept_query = deepcopy(PROV_QUERY)
accept_query["query"]["bool"]["must"][0]["term"][
"resource_id.exact"] = latest.id
accept_query["query"]["bool"]["must"][1]["term"][
"action.exact"] = "status:accepted"
provs = Provenance.q2obj(q=accept_query)
if len(provs) > 0:
last_accept = provs[0].last_updated
out_applications.writerow([
journal.id, journal.created_date,
journal.last_update_request, latest.id,
latest.last_updated, latest.application_status, diff,
last_edit, last_accept
])
# was the journal (in doaj) created before the application by greater than the threshold, and is it in a state other than rejected
if mdiff < -1 * THRESHOLD and latest.application_status != constants.APPLICATION_STATUS_REJECTED and journal.is_in_doaj(
):
out_reapps.writerow([
journal.id, journal.created_date,
journal.last_update_request, latest.id,
latest.created_date, latest.last_updated,
latest.last_manual_update, latest.application_status, mdiff
])
# now figure out if the account is missing
owner = journal.owner
if owner is None:
out_accounts.writerow([
journal.id, journal.created_date,
journal.last_update_request,
str(journal.is_in_doaj()), "NO OWNER"
])
else:
acc = Account.pull(owner)
if acc is None:
out_accounts.writerow([
journal.id, journal.created_date,
journal.last_update_request,
str(journal.is_in_doaj()), owner
])
print "processed", counter, "journals"
def create_edit_cases():
application_source = ApplicationFixtureFactory.make_application_source()
account_source = AccountFixtureFactory.make_publisher_source()
editable_application = Suggestion(**application_source)
editable_application.set_application_status(
constants.APPLICATION_STATUS_UPDATE_REQUEST)
non_editable_application = Suggestion(**application_source)
non_editable_application.set_application_status(
constants.APPLICATION_STATUS_READY)
owner_account = Account(**deepcopy(account_source))
owner_account.set_id(editable_application.owner)
non_owner_publisher = Account(**deepcopy(account_source))
non_publisher = Account(**deepcopy(account_source))
non_publisher.remove_role("publisher")
admin = Account(**deepcopy(account_source))
admin.add_role("admin")
return [
param("no_app_no_account",
None,
None,
raises=exceptions.ArgumentException),
param("no_app_with_account",
None,
owner_account,
raises=exceptions.ArgumentException),
param("app_no_account",
editable_application,
None,
raises=exceptions.ArgumentException),
param("editable_app_owning_account",
editable_application,
owner_account,
expected=True),
param("editable_app_nonowning_account",
editable_application,
non_owner_publisher,
raises=exceptions.AuthoriseException),
param("editable_app_non_publisher_account",
editable_application,
non_publisher,
raises=exceptions.AuthoriseException),
param("editable_app_admin_account",
editable_application,
admin,
expected=True),
param("non_editable_app_owning_account",
non_editable_application,
owner_account,
raises=exceptions.AuthoriseException),
param("non_editable_app_nonowning_account",
non_editable_application,
non_owner_publisher,
raises=exceptions.AuthoriseException),
param("non_editable_app_non_publisher_account",
non_editable_application,
non_publisher,
raises=exceptions.AuthoriseException),
param("non_editable_app_admin_account",
non_editable_application,
admin,
expected=True)
]
def test_01_delete_application(self, name, application_type, account_type,
current_journal, related_journal, raises):
###############################################
## set up
# create the test application (if needed), and the associated current_journal and related_journal in suitable states
application = None
cj = None
rj = None
if application_type == "found" or application_type == "locked":
application = Suggestion(
**ApplicationFixtureFactory.make_application_source())
if current_journal == "none":
application.remove_current_journal()
elif current_journal == "not_found":
application.set_current_journal("123456789987654321")
elif current_journal == "found":
cj = Journal(**JournalFixtureFactory.make_journal_source())
cj.set_id(cj.makeid())
cj.save(blocking=True)
application.set_current_journal(cj.id)
elif current_journal == "locked":
cj = Journal(**JournalFixtureFactory.make_journal_source())
cj.set_id(cj.makeid())
cj.save(blocking=True)
application.set_current_journal(cj.id)
lock.lock(constants.LOCK_JOURNAL, cj.id, "otheruser")
if related_journal == "none":
application.remove_related_journal()
elif related_journal == "not_found":
application.set_related_journal("123456789987654321")
elif related_journal == "found":
rj = Journal(**JournalFixtureFactory.make_journal_source())
rj.set_id(rj.makeid())
rj.save(blocking=True)
application.set_related_journal(rj.id)
elif related_journal == "locked":
rj = Journal(**JournalFixtureFactory.make_journal_source())
rj.set_id(rj.makeid())
rj.save(blocking=True)
application.set_related_journal(rj.id)
lock.lock(constants.LOCK_JOURNAL, rj.id, "otheruser")
acc = None
if account_type != "none":
acc = Account(**AccountFixtureFactory.make_publisher_source())
if account_type == "not_permitted":
acc.remove_role("publisher")
if application_type == "locked":
thelock = lock.lock(constants.LOCK_APPLICATION, application.id,
"otheruser")
# we can't explicitly block on the lock, but we can halt until we confirm it is saved
thelock.blockall([(thelock.id, thelock.last_updated)])
application_id = None
if application is not None:
if acc is not None:
application.set_owner(acc.id)
application.save(blocking=True)
application_id = application.id
elif application_type == "not_found":
application_id = "sdjfasofwefkwflkajdfasjd"
###########################################################
# Execution
svc = DOAJ.applicationService()
if raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.delete_application(application_id, acc)
time.sleep(1)
check_locks(application, cj, rj, acc)
else:
svc.delete_application(application_id, acc)
# we need to sleep, so the index catches up
time.sleep(1)
# check that no locks remain set for this user
check_locks(application, cj, rj, acc)
# check that the application actually is gone
if application is not None:
assert Suggestion.pull(application.id) is None
# check that the current journal no longer has a reference to the application
if cj is not None:
cj = Journal.pull(cj.id)
assert cj.current_application is None
# check that the related journal has a record that the application was deleted
if rj is not None:
rj = Journal.pull(rj.id)
record = rj.related_application_record(application.id)
assert "status" in record
assert record["status"] == "deleted"
def journals_applications_provenance(outfile_applications, outfile_accounts, outfile_reapps, conn):
with codecs.open(outfile_applications, "wb", "utf-8") as f, codecs.open(outfile_accounts, "wb", "utf-8") as g, codecs.open(outfile_reapps, "wb", "utf-8") as h:
out_applications = csv.writer(f)
out_applications.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "Application ID", "Application Last Updated", "Application Status", "Published Diff", "Latest Edit Recorded", "Latest Accepted Recorded"])
out_accounts = csv.writer(g)
out_accounts.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "In DOAJ", "Missing Account ID"])
out_reapps = csv.writer(h)
out_reapps.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "Application ID", "Application Created", "Application Last Updated", "Application Last Manual Update", "Application Status", "Published Diff"])
counter = 0
for result in esprit.tasks.scroll(conn, "journal", keepalive="45m"):
counter += 1
journal = Journal(**result)
print counter, journal.id
# first figure out if there is a broken related application
issns = journal.bibjson().issns()
applications = Suggestion.find_by_issn(issns)
latest = None
for application in applications:
if latest is None:
latest = application
if application.last_updated_timestamp > latest.last_updated_timestamp:
latest = application
if latest is None:
continue
jcreated = journal.created_timestamp
reapp = journal.last_update_request
print counter, journal.id, reapp
if reapp is not None:
jcreated = datetime.strptime(reapp, "%Y-%m-%dT%H:%M:%SZ")
jcreated = adjust_timestamp(jcreated, JOURNAL_TIMEZONE_CUTOFF)
app_lustamp = adjust_timestamp(latest.last_updated_timestamp, APP_TIMEZONE_CUTOFF)
# app_man_lustamp = latest.last_manual_update_timestamp # no need to adjust this one
app_man_lustamp = adjust_timestamp(latest.last_manual_update_timestamp, APP_TIMEZONE_CUTOFF)
td = jcreated - app_lustamp
mtd = jcreated - app_man_lustamp
diff = td.total_seconds()
mdiff = mtd.total_seconds()
# was the journal created after the application by greater than the threshold?
if diff > THRESHOLD:
last_edit = ""
last_accept = ""
edit_query = deepcopy(PROV_QUERY)
edit_query["query"]["bool"]["must"][0]["term"]["resource_id.exact"] = latest.id
edit_query["query"]["bool"]["must"][1]["term"]["action.exact"] = "edit"
provs = Provenance.q2obj(q=edit_query)
if len(provs) > 0:
last_edit = provs[0].last_updated
accept_query = deepcopy(PROV_QUERY)
accept_query["query"]["bool"]["must"][0]["term"]["resource_id.exact"] = latest.id
accept_query["query"]["bool"]["must"][1]["term"]["action.exact"] = "status:accepted"
provs = Provenance.q2obj(q=accept_query)
if len(provs) > 0:
last_accept = provs[0].last_updated
out_applications.writerow([journal.id, journal.created_date, journal.last_update_request, latest.id, latest.last_updated, latest.application_status, diff, last_edit, last_accept])
# was the journal (in doaj) created before the application by greater than the threshold, and is it in a state other than rejected
if mdiff < -1 * THRESHOLD and latest.application_status != constants.APPLICATION_STATUS_REJECTED and journal.is_in_doaj():
out_reapps.writerow([journal.id, journal.created_date, journal.last_update_request, latest.id, latest.created_date, latest.last_updated, latest.last_manual_update, latest.application_status, mdiff])
# now figure out if the account is missing
owner = journal.owner
if owner is None:
out_accounts.writerow([journal.id, journal.created_date, journal.last_update_request, str(journal.is_in_doaj()), "NO OWNER"])
else:
acc = Account.pull(owner)
if acc is None:
out_accounts.writerow([journal.id, journal.created_date, journal.last_update_request, str(journal.is_in_doaj()), owner])
print "processed", counter, "journals"
def test_01_reject_application(self,
name,
application,
application_status,
account,
prov,
current_journal,
note,
save,
raises=None):
#######################################
## set up
if save == "fail":
Suggestion.save = mock_save_fail
ap = None
journal = None
if application == "exists":
ap = Suggestion(
**ApplicationFixtureFactory.make_application_source())
ap.set_application_status(application_status)
ap.set_id(ap.makeid())
ap.remove_notes()
if current_journal == "yes":
journal = Journal(**JournalFixtureFactory.make_journal_source(
in_doaj=True))
journal.set_id(journal.makeid())
journal.set_current_application(ap.id)
journal.save(blocking=True)
ap.set_current_journal(journal.id)
else:
ap.remove_current_journal()
acc = None
if account == "publisher":
acc = Account(**AccountFixtureFactory.make_publisher_source())
elif account == "admin":
acc = Account(
**AccountFixtureFactory.make_managing_editor_source())
provenance = None
if prov != "none":
provenance = prov == "true"
thenote = None
if note == "yes":
thenote = "abcdefg"
########################################
## execute
svc = DOAJ.applicationService()
if raises is not None and raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.reject_application(ap, acc, provenance, note=thenote)
else:
svc.reject_application(ap, acc, provenance, note=thenote)
time.sleep(1)
#######################################
## Check
ap2 = Suggestion.pull(ap.id)
assert ap2 is not None
assert ap2.application_status == constants.APPLICATION_STATUS_REJECTED
assert ap2.current_journal is None
# check the updated and manually updated date are essentially the same (they can theoretically differ
# by a small amount just based on when they are set)
updated_spread = abs(
(ap2.last_updated_timestamp -
ap2.last_manual_update_timestamp).total_seconds())
assert updated_spread <= 1.0
if current_journal == "yes" and journal is not None:
j2 = Journal.pull(journal.id)
assert j2 is not None
assert j2.current_application is None
assert ap2.related_journal == j2.id
if prov == "true":
pr = Provenance.get_latest_by_resource_id(ap.id)
assert pr is not None
if note == "yes":
assert len(ap2.notes) == 1
assert ap2.notes[0].get("note") == "abcdefg"
elif note == "no":
assert len(ap2.notes) == 0
def test_02_application_2_journal(self, name, application_type, manual_update_arg, app_key_properties, current_journal, raises):
# set up for the test
#########################################
cj = None
has_seal = bool(randint(0, 1))
application = None
if application_type == "present":
application = Suggestion(**ApplicationFixtureFactory.make_application_source())
application.set_id(application.makeid())
application.remove_contacts()
application.remove_editor_group()
application.remove_editor()
application.remove_owner()
application.remove_current_journal()
application.remove_notes()
if app_key_properties == "yes":
application.add_contact("Application", "*****@*****.**")
application.set_editor_group("appeditorgroup")
application.set_editor("appeditor")
application.set_owner("appowner")
application.set_seal(has_seal)
application.add_note("Application Note")
if current_journal == "present":
journal = Journal(**JournalFixtureFactory.make_journal_source())
journal.remove_contacts()
journal.add_contact("Journal", "*****@*****.**")
journal.set_editor_group("journaleditorgroup")
journal.set_editor("journaleditor")
journal.set_owner("journalowner")
journal.remove_current_application()
journal.remove_notes()
journal.add_note("Journal Note")
journal.save(blocking=True)
application.set_current_journal(journal.id)
cj = journal
elif current_journal == "missing":
application.set_current_journal("123456789987654321")
manual_update = None
if manual_update_arg == "true":
manual_update = True
elif manual_update_arg == "false":
manual_update = False
# execute the test
########################################
svc = DOAJ.applicationService()
if raises is not None and raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.application_2_journal(application, manual_update)
else:
journal = svc.application_2_journal(application, manual_update)
# check the result
######################################
assert journal is not None
assert isinstance(journal, Journal)
assert journal.is_in_doaj() is True
jbj = journal.bibjson().data
del jbj["active"]
assert jbj == application.bibjson().data
if current_journal == "present":
assert len(journal.related_applications) == 3
else:
assert len(journal.related_applications) == 1
related = journal.related_application_record(application.id)
assert related is not None
if manual_update_arg == "true":
assert journal.last_manual_update is not None and journal.last_manual_update != "1970-01-01T00:00:00Z"
if app_key_properties == "yes":
contacts = journal.contacts()
assert len(contacts) == 1
assert contacts[0].get("name") == "Application"
assert contacts[0].get("email") == "*****@*****.**"
assert journal.editor_group == "appeditorgroup"
assert journal.editor == "appeditor"
assert journal.owner == "appowner"
assert journal.has_seal() == has_seal
if current_journal == "present":
assert len(journal.notes) == 2
else:
assert len(journal.notes) == 1
elif app_key_properties == "no":
if current_journal == "present":
contacts = journal.contacts()
assert len(contacts) == 1
assert contacts[0].get("name") == "Journal"
assert contacts[0].get("email") == "*****@*****.**"
assert journal.editor_group == "journaleditorgroup"
assert journal.editor == "journaleditor"
assert journal.owner == "journalowner"
assert journal.has_seal() == has_seal
assert len(journal.notes) == 2
elif current_journal == "none" or current_journal == "missing":
contacts = journal.contacts()
assert len(contacts) == 0
assert journal.editor_group is None
assert journal.editor is None
assert journal.owner is None
assert journal.has_seal() == has_seal
assert len(journal.notes) == 1
if current_journal == "present":
assert cj.id == journal.id
assert cj.created_date == journal.created_date
conn = esprit.raw.Connection(source.get("host"), source.get("index"))
with open("application_2_journal.csv", "wb") as f:
writer = csv.writer(f)
writer.writerow([
"application", "app_created", "app_last_update", "app_last_manual_update", "app_adjusted_lmu", "app_issns",
"journal_matches", "journal", "journal_created", "journal_reapp", "journal_issns",
"jc_ac_diff", "jc_lmua_diff", "mc1", "lra_ac_diff", "mc2",
"is_match", "reason"
])
# first, get each application and consider it
counter = 0
for result in esprit.tasks.scroll(conn, "suggestion", keepalive="1m"):
counter += 1
application = Suggestion(**result)
application.remove_related_journal()
# find all the journals that this application could be associated with (which we need to do by issn)
issns = application.bibjson().issns()
# query by each issn individually, because we're looking for the widest possible map. Querying by
# both would require both issns match
related_journals = []
related_journal_ids = []
for issn in issns:
journals = Journal.find_by_issn(issn)
for journal in journals:
if journal.id not in related_journal_ids:
related_journal_ids.append(journal.id)
related_journals.append(journal)
def applications_inconsistencies(outfile_later, outfile_missing, conn):
with codecs.open(outfile_later, "wb",
"utf-8") as f, codecs.open(outfile_missing, "wb",
"utf-8") as g:
out_later = csv.writer(f)
out_later.writerow([
"Application ID", "Application Last Updated",
"Latest Provenance Recorded", "Difference"
])
out_missing = UnicodeWriter(g)
out_missing.writerow([
"Application ID", "Application Last Manual Update",
"Latest Provenance Record", "ISSNs", "Title"
])
counter = 0
for result in esprit.tasks.scroll(conn, "suggestion", keepalive="45m"):
counter += 1
application = Suggestion(**result)
print counter, application.id
# Part 1 - later provenance records exist
latest_prov = Provenance.get_latest_by_resource_id(application.id)
if latest_prov is not None:
lustamp = adjust_timestamp(application.last_updated_timestamp,
APP_TIMEZONE_CUTOFF)
created = latest_prov.created_date
pstamp = latest_prov.created_timestamp
td = pstamp - lustamp
diff = td.total_seconds()
if diff > THRESHOLD:
out_later.writerow([
application.id, application.last_updated, created, diff
])
# Part 2 - missing journals
if application.application_status == constants.APPLICATION_STATUS_ACCEPTED:
missing = False
# find the matching journals by issn or by title
matching_journals = Journal.find_by_issn(
application.bibjson().issns())
if len(matching_journals) == 0:
# Have another go, find by title
matching_journals = Journal.find_by_title(
application.bibjson().title)
# if there are no matching journals, it is missing.
if len(matching_journals) == 0:
missing = True
else:
# if there are matching journals, find out if any of them are in the doaj. If none, then journal is still missing
those_in_doaj = len(
[j for j in matching_journals if j.is_in_doaj()])
if those_in_doaj == 0:
missing = True
# if the journal is missing, record it
if missing:
created = ""
if latest_prov is not None:
created = latest_prov.created_date
out_missing.writerow([
application.id, application.last_manual_update,
created, " ".join(application.bibjson().issns()),
application.bibjson().title
])
print "processed", counter, "suggestions"
def test_01_update_request(self, name, journal_id, journal_lock,
account, account_role, account_is_owner,
current_applications, application_lock, application_status,
completed_applications, raises,
return_app, return_jlock, return_alock,
db_jlock, db_alock, db_app):
###############################################
## set up
# create the journal
journal = None
jid = None
if journal_id == "valid":
journal = Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True))
journal.remove_related_applications()
journal.remove_current_application()
jid = journal.id
elif journal_id == "not_in_doaj":
journal = Journal(**JournalFixtureFactory.make_journal_source(in_doaj=False))
journal.remove_related_applications()
journal.remove_current_application()
jid = journal.id
elif journal_id == "missing":
jid = uuid.uuid4().hex
acc = None
if account == "yes":
acc = Account(**AccountFixtureFactory.make_publisher_source())
if account_role == "none":
acc.remove_role("publisher")
elif account_role == "admin":
acc.remove_role("publisher")
acc.add_role("admin")
acc.set_id(acc.makeid())
if account_is_owner == "yes":
acc.set_id(journal.owner)
if journal_lock == "yes":
lock.lock("journal", jid, "someoneelse", blocking=True)
latest_app = None
current_app_count = int(current_applications)
for i in range(current_app_count):
app = Suggestion(**ApplicationFixtureFactory.make_application_source())
app.set_id(app.makeid())
app.set_created("198" + str(i) + "-01-01T00:00:00Z")
app.set_current_journal(jid)
app.save()
latest_app = app
if journal is not None:
journal.set_current_application(app.id)
comp_app_count = int(completed_applications)
for i in range(comp_app_count):
app = Suggestion(**ApplicationFixtureFactory.make_application_source())
app.set_id(app.makeid())
app.set_created("197" + str(i) + "-01-01T00:00:00Z")
app.set_related_journal(jid)
app.save()
if journal is not None:
journal.add_related_application(app.id, date_accepted=app.created_date)
if current_app_count == 0 and comp_app_count == 0:
# save at least one record to initialise the index mapping, otherwise tests fail
app = Suggestion(**ApplicationFixtureFactory.make_application_source())
app.set_id(app.makeid())
app.save()
if application_lock == "yes":
lock.lock("suggestion", latest_app.id, "someoneelse", blocking=True)
if application_status != "n/a":
latest_app.set_application_status(application_status)
latest_app.save(blocking=True)
# finally save the journal record, ensuring we get a blocking save, so everything
# above here should be synchronised with the repo
if journal is not None:
journal.save(blocking=True)
###########################################################
# Execution
svc = DOAJ.applicationService()
if raises != "":
with self.assertRaises(EXCEPTIONS[raises]):
svc.update_request_for_journal(jid, acc)
else:
application, jlock, alock = svc.update_request_for_journal(jid, acc)
# we need to sleep, so the index catches up
time.sleep(1)
if return_app == "none":
assert application is None
elif return_app == "yes":
assert application is not None
if return_jlock == "none":
assert jlock is None
elif return_jlock == "yes":
assert jlock is not None
if return_alock == "none":
assert alock is None
elif return_alock == "yes":
assert alock is not None
if db_jlock == "no" and acc is not None:
assert not lock.has_lock("journal", jid, acc.id)
elif db_jlock == "yes" and acc is not None:
assert lock.has_lock("journal", jid, acc.id)
if db_alock == "no" and application.id is not None and acc is not None:
assert not lock.has_lock("suggestion", application.id, acc.id)
elif db_alock == "yes" and application.id is not None and acc is not None:
assert lock.has_lock("suggestion", application.id, acc.id)
if db_app == "no" and application.id is not None:
indb = Suggestion.q2obj(q="id.exact:" + application.id)
assert indb is None
elif db_app == "yes" and application.id is not None:
indb = Suggestion.q2obj(q="id.exact:" + application.id)
assert indb is not None
if current_app_count == 0 and comp_app_count == 0 and application is not None:
assert application.article_metadata is None
assert application.articles_last_year is None
elif application is not None:
assert application.article_metadata is not None
assert application.articles_last_year is not None