def migrate_suggestion(data):
    if "bibjson" not in data:
        return Suggestion(**data)
    ap = data.get("bibjson").get("archiving_policy")
    if ap is None:
        return Suggestion(**data)

    data["bibjson"]["archiving_policy"] = _reformat_data(ap)
    return Suggestion(**data)
def applications_inconsistencies(outfile_later, outfile_missing, conn):
    with codecs.open(outfile_later, "wb", "utf-8") as f, codecs.open(outfile_missing, "wb", "utf-8") as g:

        out_later = csv.writer(f)
        out_later.writerow(["Application ID", "Application Last Updated", "Latest Provenance Recorded", "Difference"])

        out_missing = UnicodeWriter(g)
        out_missing.writerow(["Application ID", "Application Last Manual Update", "Latest Provenance Record", "ISSNs", "Title"])

        counter = 0
        for result in esprit.tasks.scroll(conn, "suggestion", keepalive="45m"):
            counter += 1
            application = Suggestion(**result)
            print counter, application.id

            # Part 1 - later provenance records exist
            latest_prov = Provenance.get_latest_by_resource_id(application.id)
            if latest_prov is not None:
                lustamp = adjust_timestamp(application.last_updated_timestamp, APP_TIMEZONE_CUTOFF)
                created = latest_prov.created_date
                pstamp = latest_prov.created_timestamp
                td = pstamp - lustamp
                diff = td.total_seconds()

                if diff > THRESHOLD:
                    out_later.writerow([application.id, application.last_updated, created, diff])

            # Part 2 - missing journals
            if application.application_status == constants.APPLICATION_STATUS_ACCEPTED:
                missing = False

                # find the matching journals by issn or by title
                matching_journals = Journal.find_by_issn(application.bibjson().issns())
                if len(matching_journals) == 0:
                    # Have another go, find by title
                    matching_journals = Journal.find_by_title(application.bibjson().title)

                # if there are no matching journals, it is missing.
                if len(matching_journals) == 0:
                    missing = True
                else:
                    # if there are matching journals, find out if any of them are in the doaj.  If none, then journal is still missing
                    those_in_doaj = len([j for j in matching_journals if j.is_in_doaj()])
                    if those_in_doaj == 0:
                        missing = True

                # if the journal is missing, record it
                if missing:
                    created = ""
                    if latest_prov is not None:
                        created = latest_prov.created_date
                    out_missing.writerow([application.id, application.last_manual_update, created, " ".join(application.bibjson().issns()), application.bibjson().title])

        print "processed", counter, "suggestions"
Example #3
0
    def test_03_view_application(self,
                                 name,
                                 account_type,
                                 role,
                                 owner,
                                 application_type,
                                 raises=None,
                                 returns=None,
                                 auth_reason=None):
        # set up the objects
        application = None
        if application_type == "exists":
            application = Suggestion(
                **ApplicationFixtureFactory.make_application_source())

        account = None
        if account_type == "exists":
            if role == "none":
                account = Account(
                    **AccountFixtureFactory.make_publisher_source())
                account.remove_role("publisher")
            elif role == "publisher":
                account = Account(
                    **AccountFixtureFactory.make_publisher_source())
            elif role == "admin":
                account = Account(
                    **AccountFixtureFactory.make_managing_editor_source())

            if owner == "yes":
                application.set_owner(account.id)

        svc = DOAJ.authorisationService()
        if raises is not None and raises != "":
            exception = None
            with self.assertRaises(EXCEPTIONS[raises]):
                try:
                    svc.can_view_application(account, application)
                except Exception as e:
                    exception = e
                    raise e
            if raises == "AuthoriseException":
                if auth_reason == "not_owner":
                    assert exception.reason == exception.NOT_OWNER
                elif auth_reason == "wrong_role":
                    assert exception.reason == exception.WRONG_ROLE

        elif returns is not None:
            expected = returns == "true"
            assert svc.can_view_application(account, application) is expected
        else:
            assert False, "Specify either raises or returns"
Example #4
0
File: emails.py Project: DOAJ/doaj
def get_suggesters():
    q = {
        "query" : {
            "filtered" : {
                "filter" : {
                    "bool": {
                        "should" : [
                            {"terms" : {"admin.application_status.exact" : ["pending", "answer received"]}},
                            {"missing" : {"field" : "admin.application_status"}}
                        ]
                    }
                }
            }

        },
        "size" : 0,
        "facets" : {
            "suggester" : {
                "terms" : {
                    "field" : "suggestion.suggester.email.exact",
                    "size" : 20000
                }
            }
        }
    }
    es = Suggestion.query(q=q)
    emails = [(term.get("term"), term.get("count")) for term in es.get("facets", {}).get("suggester", {}).get("terms", [])]
    return emails
Example #5
0
def get_suggestions(email):
    q = {
        "query": {
            "filtered": {
                "filter": {
                    "bool": {
                        "must": [{
                            "term": {
                                "suggestion.suggester.email.exact": email
                            }
                        }],
                        "should": [{
                            "terms": {
                                "admin.application_status.exact":
                                ["pending", "answer received"]
                            }
                        }, {
                            "missing": {
                                "field": "admin.application_status"
                            }
                        }]
                    }
                }
            }
        }
    }
    return Suggestion.iterate(q)
Example #6
0
 def _sync_owner_to_application(self):
     if self.current_application is None:
         return
     from portality.models import Suggestion
     ca = Suggestion.pull(self.current_application)
     if ca is not None and ca.owner != self.owner:
         ca.set_owner(self.owner)
         ca.save(sync_owner=False)
Example #7
0
 def _sync_owner_to_application(self):
     if self.current_application is None:
         return
     from portality.models import Suggestion
     ca = Suggestion.pull(self.current_application)
     if ca is not None and ca.owner != self.owner:
         ca.set_owner(self.owner)
         ca.save(sync_owner=False)
Example #8
0
def load_application_cases():
    account = Account(**AccountFixtureFactory.make_publisher_source())
    account.set_id(account.makeid())

    application = Suggestion(**ApplicationFixtureFactory.make_application_source())
    application.makeid()

    wrong_id = uuid.uuid4()

    return [
        param("a_id_acc_lock", application, application.id, account, True, raises=lock.Locked),
        param("a_id_acc_nolock", application, application.id, account, False),
        param("a_id_noacc_nolock", application, application.id, None, False),
        param("a_noid_noacc_nolock", application, None, None, False, raises=exceptions.ArgumentException),
        param("a_wid_noacc_nolock", application, wrong_id, None, False),
        param("noa_id_noacc_nolock", None, application.id, None, False),
        param("noa_noid_noacc_nolock", None, None, None, False, raises=exceptions.ArgumentException)
    ]
Example #9
0
def change_status(ids, new_status):
    statuses = [x[0] for x in Choices.application_status("admin") if x[0] != ""]
    if new_status not in statuses:
        raise Exception("Must use an allowed status: " + ", ".join(statuses))

    for id in ids:
        a = Suggestion.pull(id)
        a.set_application_status(new_status)
        a.save()
def change_status(ids, new_status):
    statuses = [
        x[0] for x in Choices.application_status("admin") if x[0] != ""
    ]
    if new_status not in statuses:
        raise Exception("Must use an allowed status: " + ", ".join(statuses))

    for id in ids:
        a = Suggestion.pull(id)
        a.set_application_status(new_status)
        a.save()
Example #11
0
def traverse_applications():
    a_report = {'country': [], 'currency': [], 'language': []}

    statuses_incomplete = list(
        set(constants.APPLICATION_STATUSES_ALL).difference({
            constants.APPLICATION_STATUS_REJECTED,
            constants.APPLICATION_STATUS_ACCEPTED
        }))
    for a in Suggestion.list_by_status(statuses_incomplete):
        check_invalid_datasets(a, a_report)

    return a_report
Example #12
0
    def make_application_spread(cls, desired_output, period):
        desired_output = deepcopy(desired_output)
        header = desired_output[0]
        del desired_output[0]
        del header[0]
        ranges = []
        for h in header:
            start = None
            end = None
            if period == "month":
                startts = dates.parse(h, "%Y-%m")
                year, month = divmod(startts.month+1, 12)
                if month == 0:
                    month = 12
                    year = year - 1
                endts = datetime(startts.year + year, month, 1)
                start = dates.format(startts)
                end = dates.format(endts)
            elif period == "year":
                startts = dates.parse(h, "%Y")
                endts = datetime(startts.year + 1, 1, 1)
                start = dates.format(startts)
                end = dates.format(endts)

            ranges.append((start, end))

        apps = []
        for row in desired_output:
            country = row[0]
            del row[0]
            for i in range(len(row)):
                count = row[i]
                start, end = ranges[i]
                for j in range(count):
                    s = Suggestion()
                    s.set_created(dates.random_date(start, end))
                    s.bibjson().country = country
                    apps.append(s)

        return apps
Example #13
0
def create_edit_cases():
    application_source = ApplicationFixtureFactory.make_application_source()
    account_source = AccountFixtureFactory.make_publisher_source()

    editable_application = Suggestion(**application_source)
    editable_application.set_application_status(constants.APPLICATION_STATUS_UPDATE_REQUEST)

    non_editable_application = Suggestion(**application_source)
    non_editable_application.set_application_status(constants.APPLICATION_STATUS_READY)

    owner_account = Account(**deepcopy(account_source))
    owner_account.set_id(editable_application.owner)

    non_owner_publisher = Account(**deepcopy(account_source))

    non_publisher = Account(**deepcopy(account_source))
    non_publisher.remove_role("publisher")

    admin = Account(**deepcopy(account_source))
    admin.add_role("admin")

    return [
        param("no_app_no_account", None, None, raises=exceptions.ArgumentException),
        param("no_app_with_account", None, owner_account, raises=exceptions.ArgumentException),
        param("app_no_account", editable_application, None, raises=exceptions.ArgumentException),
        param("editable_app_owning_account", editable_application, owner_account, expected=True),
        param("editable_app_nonowning_account", editable_application, non_owner_publisher, raises=exceptions.AuthoriseException),
        param("editable_app_non_publisher_account", editable_application, non_publisher, raises=exceptions.AuthoriseException),
        param("editable_app_admin_account", editable_application, admin, expected=True),
        param("non_editable_app_owning_account", non_editable_application, owner_account, raises=exceptions.AuthoriseException),
        param("non_editable_app_nonowning_account", non_editable_application, non_owner_publisher, raises=exceptions.AuthoriseException),
        param("non_editable_app_non_publisher_account", non_editable_application, non_publisher, raises=exceptions.AuthoriseException),
        param("non_editable_app_admin_account", non_editable_application, admin, expected=True)
    ]
Example #14
0
    def test_03_view_application(self, name, account_type, role, owner, application_type, raises=None, returns=None, auth_reason=None):
        # set up the objects
        application = None
        if application_type == "exists":
            application = Suggestion(**ApplicationFixtureFactory.make_application_source())

        account = None
        if account_type == "exists":
            if role == "none":
                account = Account(**AccountFixtureFactory.make_publisher_source())
                account.remove_role("publisher")
            elif role == "publisher":
                account = Account(**AccountFixtureFactory.make_publisher_source())
            elif role == "admin":
                account = Account(**AccountFixtureFactory.make_managing_editor_source())

            if owner == "yes":
                application.set_owner(account.id)


        svc = DOAJ.authorisationService()
        if raises is not None and raises != "":
            exception = None
            with self.assertRaises(EXCEPTIONS[raises]):
                try:
                    svc.can_view_application(account, application)
                except Exception as e:
                    exception = e
                    raise e
            if raises == "AuthoriseException":
                if auth_reason == "not_owner":
                    assert exception.reason == exception.NOT_OWNER
                elif auth_reason == "wrong_role":
                    assert exception.reason == exception.WRONG_ROLE

        elif returns is not None:
            expected = returns == "true"
            assert svc.can_view_application(account, application) is expected
        else:
            assert False, "Specify either raises or returns"
Example #15
0
def migrate_suggestions(source):
    # read in the content
    f = open(source)
    xml = etree.parse(f)
    f.close()
    suggestions = xml.getroot()
    print "migrating", str(len(suggestions)), "suggestion records"

    for element in suggestions:
        s = Suggestion()
        
        # re-use the journal bibjson crosswalk
        cb = _to_journal_bibjson(element)
        s.set_bibjson(cb)
        
        # explicitly set the open-access-ness
        cb.set_open_access(_is_open_access(element))
        
        # suggestion info
        _to_suggestion(element, s)
        
        s.save()
Example #16
0
File: emails.py Project: DOAJ/doaj
def get_suggestions(email):
    q = {
        "query" : {
            "filtered" : {
                "filter" : {
                    "bool" : {
                        "must" : [{"term" : {"suggestion.suggester.email.exact" : email}}],
                        "should" : [
                            {"terms" : {"admin.application_status.exact" : ["pending", "answer received"]}},
                            {"missing" : {"field" : "admin.application_status"}}
                        ]
                    }
                }
            }
        }
    }
    return Suggestion.iterate(q)
Example #17
0
def get_suggesters():
    q = {
        "query": {
            "filtered": {
                "filter": {
                    "bool": {
                        "should": [{
                            "terms": {
                                "admin.application_status.exact":
                                ["pending", "answer received"]
                            }
                        }, {
                            "missing": {
                                "field": "admin.application_status"
                            }
                        }]
                    }
                }
            }
        },
        "size": 0,
        "facets": {
            "suggester": {
                "terms": {
                    "field": "suggestion.suggester.email.exact",
                    "size": 20000
                }
            }
        }
    }
    es = Suggestion.query(q=q)
    emails = [
        (term.get("term"), term.get("count"))
        for term in es.get("facets", {}).get("suggester", {}).get("terms", [])
    ]
    return emails
Example #18
0
def migrate_suggestions(source):
    # read in the content
    f = open(source)
    xml = etree.parse(f)
    f.close()
    suggestions = xml.getroot()
    print "migrating", str(len(suggestions)), "suggestion records"

    for element in suggestions:
        s = Suggestion()
        
        # re-use the journal bibjson crosswalk
        cb = _to_journal_bibjson(element)
        s.set_bibjson(cb)
        
        # explicitly set the open-access-ness
        cb.set_open_access(_is_open_access(element))
        
        # suggestion info
        _to_suggestion(element, s)
        
        s.save()
    def test_02_application_2_journal(self, name, application_type,
                                      manual_update_arg, app_key_properties,
                                      current_journal, raises):
        # set up for the test
        #########################################

        cj = None
        has_seal = bool(randint(0, 1))
        application = None
        if application_type == "present":
            application = Suggestion(
                **ApplicationFixtureFactory.make_application_source())
            application.set_id(application.makeid())
            application.remove_contacts()
            application.remove_editor_group()
            application.remove_editor()
            application.remove_owner()
            application.remove_current_journal()
            application.remove_notes()

            if app_key_properties == "yes":
                application.add_contact("Application",
                                        "*****@*****.**")
                application.set_editor_group("appeditorgroup")
                application.set_editor("appeditor")
                application.set_owner("appowner")

            application.set_seal(has_seal)
            application.add_note("Application Note")

            if current_journal == "present":
                journal = Journal(
                    **JournalFixtureFactory.make_journal_source())
                journal.remove_contacts()
                journal.add_contact("Journal", "*****@*****.**")
                journal.set_editor_group("journaleditorgroup")
                journal.set_editor("journaleditor")
                journal.set_owner("journalowner")
                journal.remove_current_application()
                journal.remove_notes()
                journal.add_note("Journal Note")
                journal.save(blocking=True)
                application.set_current_journal(journal.id)
                cj = journal
            elif current_journal == "missing":
                application.set_current_journal("123456789987654321")

        manual_update = None
        if manual_update_arg == "true":
            manual_update = True
        elif manual_update_arg == "false":
            manual_update = False

        # execute the test
        ########################################

        svc = DOAJ.applicationService()
        if raises is not None and raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.application_2_journal(application, manual_update)
        else:
            journal = svc.application_2_journal(application, manual_update)

            # check the result
            ######################################

            assert journal is not None
            assert isinstance(journal, Journal)
            assert journal.is_in_doaj() is True

            jbj = journal.bibjson().data
            del jbj["active"]
            assert jbj == application.bibjson().data

            if current_journal == "present":
                assert len(journal.related_applications) == 3
            else:
                assert len(journal.related_applications) == 1
            related = journal.related_application_record(application.id)
            assert related is not None

            if manual_update_arg == "true":
                assert journal.last_manual_update is not None and journal.last_manual_update != "1970-01-01T00:00:00Z"

            if app_key_properties == "yes":
                contacts = journal.contacts()
                assert len(contacts) == 1
                assert contacts[0].get("name") == "Application"
                assert contacts[0].get("email") == "*****@*****.**"
                assert journal.editor_group == "appeditorgroup"
                assert journal.editor == "appeditor"
                assert journal.owner == "appowner"
                assert journal.has_seal() == has_seal

                if current_journal == "present":
                    assert len(journal.notes) == 2
                else:
                    assert len(journal.notes) == 1

            elif app_key_properties == "no":
                if current_journal == "present":
                    contacts = journal.contacts()
                    assert len(contacts) == 1
                    assert contacts[0].get("name") == "Journal"
                    assert contacts[0].get("email") == "*****@*****.**"
                    assert journal.editor_group == "journaleditorgroup"
                    assert journal.editor == "journaleditor"
                    assert journal.owner == "journalowner"
                    assert journal.has_seal() == has_seal
                    assert len(journal.notes) == 2

                elif current_journal == "none" or current_journal == "missing":
                    contacts = journal.contacts()
                    assert len(contacts) == 0
                    assert journal.editor_group is None
                    assert journal.editor is None
                    assert journal.owner is None
                    assert journal.has_seal() == has_seal
                    assert len(journal.notes) == 1

            if current_journal == "present":
                assert cj.id == journal.id
                assert cj.created_date == journal.created_date
Example #20
0
    def test_01_delete_application(self, name, application_type, account_type, current_journal, related_journal, raises):

        ###############################################
        ## set up

        # create the test application (if needed), and the associated current_journal and related_journal in suitable states
        application = None
        cj = None
        rj = None
        if application_type == "found" or application_type == "locked":
            application = Suggestion(**ApplicationFixtureFactory.make_application_source())

            if current_journal == "none":
                application.remove_current_journal()
            elif current_journal == "not_found":
                application.set_current_journal("123456789987654321")
            elif current_journal == "found":
                cj = Journal(**JournalFixtureFactory.make_journal_source())
                cj.set_id(cj.makeid())
                cj.save(blocking=True)
                application.set_current_journal(cj.id)
            elif current_journal == "locked":
                cj = Journal(**JournalFixtureFactory.make_journal_source())
                cj.set_id(cj.makeid())
                cj.save(blocking=True)
                application.set_current_journal(cj.id)
                lock.lock(constants.LOCK_JOURNAL, cj.id, "otheruser")

            if related_journal == "none":
                application.remove_related_journal()
            elif related_journal == "not_found":
                application.set_related_journal("123456789987654321")
            elif related_journal == "found":
                rj = Journal(**JournalFixtureFactory.make_journal_source())
                rj.set_id(rj.makeid())
                rj.save(blocking=True)
                application.set_related_journal(rj.id)
            elif related_journal == "locked":
                rj = Journal(**JournalFixtureFactory.make_journal_source())
                rj.set_id(rj.makeid())
                rj.save(blocking=True)
                application.set_related_journal(rj.id)
                lock.lock(constants.LOCK_JOURNAL, rj.id, "otheruser")


        acc = None
        if account_type != "none":
            acc = Account(**AccountFixtureFactory.make_publisher_source())
            if account_type == "not_permitted":
                acc.remove_role("publisher")
            if application_type == "locked":
                thelock = lock.lock(constants.LOCK_APPLICATION, application.id, "otheruser")
                # we can't explicitly block on the lock, but we can halt until we confirm it is saved
                thelock.blockall([(thelock.id, thelock.last_updated)])

        application_id = None
        if application is not None:
            if acc is not None:
                application.set_owner(acc.id)
            application.save(blocking=True)
            application_id = application.id
        elif application_type == "not_found":
            application_id = u"sdjfasofwefkwflkajdfasjd"

        ###########################################################
        # Execution

        svc = DOAJ.applicationService()
        if raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.delete_application(application_id, acc)
            time.sleep(1)
            check_locks(application, cj, rj, acc)
        else:
            svc.delete_application(application_id, acc)

            # we need to sleep, so the index catches up
            time.sleep(1)

            # check that no locks remain set for this user
            check_locks(application, cj, rj, acc)

            # check that the application actually is gone
            if application is not None:
                assert Suggestion.pull(application.id) is None

            # check that the current journal no longer has a reference to the application
            if cj is not None:
                cj = Journal.pull(cj.id)
                assert cj.current_application is None

            # check that the related journal has a record that the application was deleted
            if rj is not None:
                rj = Journal.pull(rj.id)
                record = rj.related_application_record(application.id)
                assert "status" in record
                assert record["status"] == "deleted"
Example #21
0
with open("application_2_journal.csv", "wb") as f:
    writer = csv.writer(f)
    writer.writerow([
        "application", "app_created", "app_last_update",
        "app_last_manual_update", "app_adjusted_lmu", "app_issns",
        "journal_matches", "journal", "journal_created", "journal_reapp",
        "journal_issns", "jc_ac_diff", "jc_lmua_diff", "mc1", "lra_ac_diff",
        "mc2", "is_match", "reason"
    ])

    # first, get each application and consider it
    counter = 0
    for result in esprit.tasks.scroll(conn, "suggestion", keepalive="1m"):
        counter += 1
        application = Suggestion(**result)
        application.remove_related_journal()

        # find all the journals that this application could be associated with (which we need to do by issn)
        issns = application.bibjson().issns()

        # query by each issn individually, because we're looking for the widest possible map.  Querying by
        # both would require both issns match
        related_journals = []
        related_journal_ids = []
        for issn in issns:
            journals = Journal.find_by_issn(issn)
            for journal in journals:
                if journal.id not in related_journal_ids:
                    related_journal_ids.append(journal.id)
                    related_journals.append(journal)
    def test_01_accept_application(self, name, application_type, account_type, manual_update, provenance, raises, result_provenance, result_manual_update):

        ###############################################
        ## set up

        # create the application
        application = None
        if application_type == "save_fail":
            application = Suggestion(**ApplicationFixtureFactory.make_application_source())
            application.save = mock_save
            Journal.save = mock_save
        elif application_type == "with_current_journal":
            application = Suggestion(**ApplicationFixtureFactory.make_application_source())
            application.remove_notes()
            application.add_note("unique 1", "2002-01-01T00:00:00Z")
            application.add_note("duplicate", "2001-01-01T00:00:00Z")
            cj = application.current_journal
            journal = Journal(**JournalFixtureFactory.make_journal_source())
            journal.set_id(cj)
            journal.remove_notes()
            journal.add_note("unique 2", "2003-01-01T00:00:00Z")
            journal.add_note("duplicate", "2001-01-01T00:00:00Z")
            journal.save(blocking=True)
        elif application_type == "no_current_journal":
            application = Suggestion(**ApplicationFixtureFactory.make_application_source())
            application.remove_current_journal()


        acc = None
        if account_type == "not_allowed":
            acc = Account(**AccountFixtureFactory.make_publisher_source())
        elif account_type == "allowed":
            acc = Account(**AccountFixtureFactory.make_managing_editor_source())

        mu = None
        if manual_update in ["true", "false"]:
            mu = manual_update == "true"

        prov = None
        if provenance in ["true", "false"]:
            prov = provenance == "true"

        save = bool(randint(0,1))

        ###########################################################
        # Execution

        svc = DOAJ.applicationService()
        if raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.accept_application(application, acc, mu, prov)
        else:
            journal = svc.accept_application(application, acc, mu, prov, save_journal=save, save_application=save)

            # we need to sleep, so the index catches up
            time.sleep(1)

            # check a few common things
            assert application.application_status == constants.APPLICATION_STATUS_ACCEPTED
            assert application.current_journal is None
            assert journal.current_application is None
            assert application.related_journal == journal.id

            related = journal.related_applications
            if application_type == "with_current_journal":
                assert len(related) == 3
            elif application_type == "no_current_journal":
                assert len(related) == 1
            assert related[0].get("application_id") == application.id
            assert related[0].get("date_accepted") is not None

            if result_manual_update == "yes":
                assert journal.last_manual_update is not None
                assert journal.last_manual_update != "1970-01-01T00:00:00Z"
                assert application.last_manual_update is not None
                assert application.last_manual_update != "1970-01-01T00:00:00Z"
            elif result_manual_update == "no":
                assert journal.last_manual_update is None
                assert application.last_manual_update is None

            if application_type == "with_current_journal":
                assert len(journal.notes) == 3
                notevals = [note.get("note") for note in journal.notes]
                assert "duplicate" in notevals
                assert "unique 1" in notevals
                assert "unique 2" in notevals

            app_prov = Provenance.get_latest_by_resource_id(application.id)
            if result_provenance == "yes":
                assert app_prov is not None
            elif result_provenance == "no":
                assert app_prov is None

            if save:
                pass
Example #23
0
    def test_01_reject_application(self, name, application, application_status, account, prov, current_journal, note, save, raises=None):

        #######################################
        ## set up

        if save == "fail":
            Suggestion.save = mock_save_fail

        ap = None
        journal = None
        if application == "exists":
            ap = Suggestion(**ApplicationFixtureFactory.make_application_source())
            ap.set_application_status(application_status)
            ap.set_id(ap.makeid())
            ap.remove_notes()
            if current_journal == "yes":
                journal = Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True))
                journal.set_id(journal.makeid())
                journal.set_current_application(ap.id)
                journal.save(blocking=True)
                ap.set_current_journal(journal.id)
            else:
                ap.remove_current_journal()

        acc = None
        if account == "publisher":
            acc = Account(**AccountFixtureFactory.make_publisher_source())
        elif account == "admin":
            acc = Account(**AccountFixtureFactory.make_managing_editor_source())

        provenance = None
        if prov != "none":
            provenance = prov == "true"

        thenote = None
        if note == "yes":
            thenote = "abcdefg"

        ########################################
        ## execute

        svc = DOAJ.applicationService()
        if raises is not None and raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.reject_application(ap, acc, provenance, note=thenote)
        else:
            svc.reject_application(ap, acc, provenance, note=thenote)
            time.sleep(1)

            #######################################
            ## Check

            ap2 = Suggestion.pull(ap.id)
            assert ap2 is not None
            assert ap2.application_status == constants.APPLICATION_STATUS_REJECTED
            assert ap2.current_journal is None

            # check the updated and manually updated date are essentially the same (they can theoretically differ
            # by a small amount just based on when they are set)
            updated_spread = abs((ap2.last_updated_timestamp - ap2.last_manual_update_timestamp).total_seconds())
            assert updated_spread <= 1.0

            if current_journal == "yes" and journal is not None:
                j2 = Journal.pull(journal.id)
                assert j2 is not None
                assert j2.current_application is None
                assert ap2.related_journal == j2.id

            if prov == "true":
                pr = Provenance.get_latest_by_resource_id(ap.id)
                assert pr is not None

            if note == "yes":
                assert len(ap2.notes) == 1
                assert ap2.notes[0].get("note") == "abcdefg"
            elif note == "no":
                assert len(ap2.notes) == 0
Example #24
0
    def test_01_update_request(self, name, journal_id, journal_lock, account,
                               account_role, account_is_owner,
                               current_applications, application_lock,
                               application_status, completed_applications,
                               raises, return_app, return_jlock, return_alock,
                               db_jlock, db_alock, db_app):

        ###############################################
        ## set up

        # create the journal
        journal = None
        jid = None
        if journal_id == "valid":
            journal = Journal(**JournalFixtureFactory.make_journal_source(
                in_doaj=True))
            journal.remove_related_applications()
            journal.remove_current_application()
            jid = journal.id
        elif journal_id == "not_in_doaj":
            journal = Journal(**JournalFixtureFactory.make_journal_source(
                in_doaj=False))
            journal.remove_related_applications()
            journal.remove_current_application()
            jid = journal.id
        elif journal_id == "missing":
            jid = uuid.uuid4().hex

        acc = None
        if account == "yes":
            acc = Account(**AccountFixtureFactory.make_publisher_source())
            if account_role == "none":
                acc.remove_role("publisher")
            elif account_role == "admin":
                acc.remove_role("publisher")
                acc.add_role("admin")
            acc.set_id(acc.makeid())
            if account_is_owner == "yes":
                acc.set_id(journal.owner)

        if journal_lock == "yes":
            lock.lock("journal", jid, "someoneelse", blocking=True)

        latest_app = None
        current_app_count = int(current_applications)
        for i in range(current_app_count):
            app = Suggestion(
                **ApplicationFixtureFactory.make_application_source())
            app.set_id(app.makeid())
            app.set_created("198" + str(i) + "-01-01T00:00:00Z")
            app.set_current_journal(jid)
            app.save()
            latest_app = app
            if journal is not None:
                journal.set_current_application(app.id)

        comp_app_count = int(completed_applications)
        for i in range(comp_app_count):
            app = Suggestion(
                **ApplicationFixtureFactory.make_application_source())
            app.set_id(app.makeid())
            app.set_created("197" + str(i) + "-01-01T00:00:00Z")
            app.set_related_journal(jid)
            app.save()
            if journal is not None:
                journal.add_related_application(app.id,
                                                date_accepted=app.created_date)

        if current_app_count == 0 and comp_app_count == 0:
            # save at least one record to initialise the index mapping, otherwise tests fail
            app = Suggestion(
                **ApplicationFixtureFactory.make_application_source())
            app.set_id(app.makeid())
            app.save()

        if application_lock == "yes":
            lock.lock("suggestion",
                      latest_app.id,
                      "someoneelse",
                      blocking=True)

        if application_status != "n/a":
            latest_app.set_application_status(application_status)
            latest_app.save(blocking=True)

        # finally save the journal record, ensuring we get a blocking save, so everything
        # above here should be synchronised with the repo
        if journal is not None:
            journal.save(blocking=True)

        ###########################################################
        # Execution

        svc = DOAJ.applicationService()
        if raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.update_request_for_journal(jid, acc)
        else:
            application, jlock, alock = svc.update_request_for_journal(
                jid, acc)

            # we need to sleep, so the index catches up
            time.sleep(1)

            if return_app == "none":
                assert application is None
            elif return_app == "yes":
                assert application is not None

            if return_jlock == "none":
                assert jlock is None
            elif return_jlock == "yes":
                assert jlock is not None

            if return_alock == "none":
                assert alock is None
            elif return_alock == "yes":
                assert alock is not None

            if db_jlock == "no" and acc is not None:
                assert not lock.has_lock("journal", jid, acc.id)
            elif db_jlock == "yes" and acc is not None:
                l = lock.has_lock("journal", jid, acc.id)
                assert lock.has_lock("journal", jid, acc.id)

            if db_alock == "no" and application.id is not None and acc is not None:
                assert not lock.has_lock("suggestion", application.id, acc.id)
            elif db_alock == "yes" and application.id is not None and acc is not None:
                assert lock.has_lock("suggestion", application.id, acc.id)

            if db_app == "no" and application.id is not None:
                indb = Suggestion.q2obj(q="id.exact:" + application.id)
                assert indb is None
            elif db_app == "yes" and application.id is not None:
                indb = Suggestion.q2obj(q="id.exact:" + application.id)
                assert indb is not None

            if current_app_count == 0 and comp_app_count == 0 and application is not None:
                assert application.article_metadata is None
                assert application.articles_last_year is None
            elif application is not None:
                assert application.article_metadata is not None
                assert application.articles_last_year is not None
def journals_applications_provenance(outfile_applications, outfile_accounts,
                                     outfile_reapps, conn):
    with codecs.open(outfile_applications, "wb", "utf-8") as f, codecs.open(
            outfile_accounts, "wb",
            "utf-8") as g, codecs.open(outfile_reapps, "wb", "utf-8") as h:
        out_applications = csv.writer(f)
        out_applications.writerow([
            "Journal ID", "Journal Created", "Journal Reapplied",
            "Application ID", "Application Last Updated", "Application Status",
            "Published Diff", "Latest Edit Recorded",
            "Latest Accepted Recorded"
        ])

        out_accounts = csv.writer(g)
        out_accounts.writerow([
            "Journal ID", "Journal Created", "Journal Reapplied", "In DOAJ",
            "Missing Account ID"
        ])

        out_reapps = csv.writer(h)
        out_reapps.writerow([
            "Journal ID", "Journal Created", "Journal Reapplied",
            "Application ID", "Application Created",
            "Application Last Updated", "Application Last Manual Update",
            "Application Status", "Published Diff"
        ])

        counter = 0
        for result in esprit.tasks.scroll(conn, "journal", keepalive="45m"):
            counter += 1
            journal = Journal(**result)
            print counter, journal.id

            # first figure out if there is a broken related application
            issns = journal.bibjson().issns()
            applications = Suggestion.find_by_issn(issns)
            latest = None
            for application in applications:
                if latest is None:
                    latest = application
                if application.last_updated_timestamp > latest.last_updated_timestamp:
                    latest = application

            if latest is None:
                continue

            jcreated = journal.created_timestamp
            reapp = journal.last_update_request
            print counter, journal.id, reapp
            if reapp is not None:
                jcreated = datetime.strptime(reapp, "%Y-%m-%dT%H:%M:%SZ")
            jcreated = adjust_timestamp(jcreated, JOURNAL_TIMEZONE_CUTOFF)

            app_lustamp = adjust_timestamp(latest.last_updated_timestamp,
                                           APP_TIMEZONE_CUTOFF)
            # app_man_lustamp = latest.last_manual_update_timestamp # no need to adjust this one
            app_man_lustamp = adjust_timestamp(
                latest.last_manual_update_timestamp, APP_TIMEZONE_CUTOFF)
            td = jcreated - app_lustamp
            mtd = jcreated - app_man_lustamp
            diff = td.total_seconds()
            mdiff = mtd.total_seconds()

            # was the journal created after the application by greater than the threshold?
            if diff > THRESHOLD:
                last_edit = ""
                last_accept = ""

                edit_query = deepcopy(PROV_QUERY)
                edit_query["query"]["bool"]["must"][0]["term"][
                    "resource_id.exact"] = latest.id
                edit_query["query"]["bool"]["must"][1]["term"][
                    "action.exact"] = "edit"
                provs = Provenance.q2obj(q=edit_query)
                if len(provs) > 0:
                    last_edit = provs[0].last_updated

                accept_query = deepcopy(PROV_QUERY)
                accept_query["query"]["bool"]["must"][0]["term"][
                    "resource_id.exact"] = latest.id
                accept_query["query"]["bool"]["must"][1]["term"][
                    "action.exact"] = "status:accepted"
                provs = Provenance.q2obj(q=accept_query)
                if len(provs) > 0:
                    last_accept = provs[0].last_updated

                out_applications.writerow([
                    journal.id, journal.created_date,
                    journal.last_update_request, latest.id,
                    latest.last_updated, latest.application_status, diff,
                    last_edit, last_accept
                ])

            # was the journal (in doaj) created before the application by greater than the threshold, and is it in a state other than rejected
            if mdiff < -1 * THRESHOLD and latest.application_status != constants.APPLICATION_STATUS_REJECTED and journal.is_in_doaj(
            ):
                out_reapps.writerow([
                    journal.id, journal.created_date,
                    journal.last_update_request, latest.id,
                    latest.created_date, latest.last_updated,
                    latest.last_manual_update, latest.application_status, mdiff
                ])

            # now figure out if the account is missing
            owner = journal.owner
            if owner is None:
                out_accounts.writerow([
                    journal.id, journal.created_date,
                    journal.last_update_request,
                    str(journal.is_in_doaj()), "NO OWNER"
                ])
            else:
                acc = Account.pull(owner)
                if acc is None:
                    out_accounts.writerow([
                        journal.id, journal.created_date,
                        journal.last_update_request,
                        str(journal.is_in_doaj()), owner
                    ])

        print "processed", counter, "journals"
Example #26
0
def create_edit_cases():
    application_source = ApplicationFixtureFactory.make_application_source()
    account_source = AccountFixtureFactory.make_publisher_source()

    editable_application = Suggestion(**application_source)
    editable_application.set_application_status(
        constants.APPLICATION_STATUS_UPDATE_REQUEST)

    non_editable_application = Suggestion(**application_source)
    non_editable_application.set_application_status(
        constants.APPLICATION_STATUS_READY)

    owner_account = Account(**deepcopy(account_source))
    owner_account.set_id(editable_application.owner)

    non_owner_publisher = Account(**deepcopy(account_source))

    non_publisher = Account(**deepcopy(account_source))
    non_publisher.remove_role("publisher")

    admin = Account(**deepcopy(account_source))
    admin.add_role("admin")

    return [
        param("no_app_no_account",
              None,
              None,
              raises=exceptions.ArgumentException),
        param("no_app_with_account",
              None,
              owner_account,
              raises=exceptions.ArgumentException),
        param("app_no_account",
              editable_application,
              None,
              raises=exceptions.ArgumentException),
        param("editable_app_owning_account",
              editable_application,
              owner_account,
              expected=True),
        param("editable_app_nonowning_account",
              editable_application,
              non_owner_publisher,
              raises=exceptions.AuthoriseException),
        param("editable_app_non_publisher_account",
              editable_application,
              non_publisher,
              raises=exceptions.AuthoriseException),
        param("editable_app_admin_account",
              editable_application,
              admin,
              expected=True),
        param("non_editable_app_owning_account",
              non_editable_application,
              owner_account,
              raises=exceptions.AuthoriseException),
        param("non_editable_app_nonowning_account",
              non_editable_application,
              non_owner_publisher,
              raises=exceptions.AuthoriseException),
        param("non_editable_app_non_publisher_account",
              non_editable_application,
              non_publisher,
              raises=exceptions.AuthoriseException),
        param("non_editable_app_admin_account",
              non_editable_application,
              admin,
              expected=True)
    ]
Example #27
0
    def test_01_delete_application(self, name, application_type, account_type,
                                   current_journal, related_journal, raises):

        ###############################################
        ## set up

        # create the test application (if needed), and the associated current_journal and related_journal in suitable states
        application = None
        cj = None
        rj = None
        if application_type == "found" or application_type == "locked":
            application = Suggestion(
                **ApplicationFixtureFactory.make_application_source())

            if current_journal == "none":
                application.remove_current_journal()
            elif current_journal == "not_found":
                application.set_current_journal("123456789987654321")
            elif current_journal == "found":
                cj = Journal(**JournalFixtureFactory.make_journal_source())
                cj.set_id(cj.makeid())
                cj.save(blocking=True)
                application.set_current_journal(cj.id)
            elif current_journal == "locked":
                cj = Journal(**JournalFixtureFactory.make_journal_source())
                cj.set_id(cj.makeid())
                cj.save(blocking=True)
                application.set_current_journal(cj.id)
                lock.lock(constants.LOCK_JOURNAL, cj.id, "otheruser")

            if related_journal == "none":
                application.remove_related_journal()
            elif related_journal == "not_found":
                application.set_related_journal("123456789987654321")
            elif related_journal == "found":
                rj = Journal(**JournalFixtureFactory.make_journal_source())
                rj.set_id(rj.makeid())
                rj.save(blocking=True)
                application.set_related_journal(rj.id)
            elif related_journal == "locked":
                rj = Journal(**JournalFixtureFactory.make_journal_source())
                rj.set_id(rj.makeid())
                rj.save(blocking=True)
                application.set_related_journal(rj.id)
                lock.lock(constants.LOCK_JOURNAL, rj.id, "otheruser")

        acc = None
        if account_type != "none":
            acc = Account(**AccountFixtureFactory.make_publisher_source())
            if account_type == "not_permitted":
                acc.remove_role("publisher")
            if application_type == "locked":
                thelock = lock.lock(constants.LOCK_APPLICATION, application.id,
                                    "otheruser")
                # we can't explicitly block on the lock, but we can halt until we confirm it is saved
                thelock.blockall([(thelock.id, thelock.last_updated)])

        application_id = None
        if application is not None:
            if acc is not None:
                application.set_owner(acc.id)
            application.save(blocking=True)
            application_id = application.id
        elif application_type == "not_found":
            application_id = "sdjfasofwefkwflkajdfasjd"

        ###########################################################
        # Execution

        svc = DOAJ.applicationService()
        if raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.delete_application(application_id, acc)
            time.sleep(1)
            check_locks(application, cj, rj, acc)
        else:
            svc.delete_application(application_id, acc)

            # we need to sleep, so the index catches up
            time.sleep(1)

            # check that no locks remain set for this user
            check_locks(application, cj, rj, acc)

            # check that the application actually is gone
            if application is not None:
                assert Suggestion.pull(application.id) is None

            # check that the current journal no longer has a reference to the application
            if cj is not None:
                cj = Journal.pull(cj.id)
                assert cj.current_application is None

            # check that the related journal has a record that the application was deleted
            if rj is not None:
                rj = Journal.pull(rj.id)
                record = rj.related_application_record(application.id)
                assert "status" in record
                assert record["status"] == "deleted"
def journals_applications_provenance(outfile_applications, outfile_accounts, outfile_reapps, conn):
    with codecs.open(outfile_applications, "wb", "utf-8") as f, codecs.open(outfile_accounts, "wb", "utf-8") as g, codecs.open(outfile_reapps, "wb", "utf-8") as h:
        out_applications = csv.writer(f)
        out_applications.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "Application ID", "Application Last Updated", "Application Status", "Published Diff", "Latest Edit Recorded", "Latest Accepted Recorded"])

        out_accounts = csv.writer(g)
        out_accounts.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "In DOAJ", "Missing Account ID"])

        out_reapps = csv.writer(h)
        out_reapps.writerow(["Journal ID", "Journal Created", "Journal Reapplied", "Application ID", "Application Created", "Application Last Updated", "Application Last Manual Update", "Application Status", "Published Diff"])

        counter = 0
        for result in esprit.tasks.scroll(conn, "journal", keepalive="45m"):
            counter += 1
            journal = Journal(**result)
            print counter, journal.id

            # first figure out if there is a broken related application
            issns = journal.bibjson().issns()
            applications = Suggestion.find_by_issn(issns)
            latest = None
            for application in applications:
                if latest is None:
                    latest = application
                if application.last_updated_timestamp > latest.last_updated_timestamp:
                    latest = application

            if latest is None:
                continue

            jcreated = journal.created_timestamp
            reapp = journal.last_update_request
            print counter, journal.id, reapp
            if reapp is not None:
                jcreated = datetime.strptime(reapp, "%Y-%m-%dT%H:%M:%SZ")
            jcreated = adjust_timestamp(jcreated, JOURNAL_TIMEZONE_CUTOFF)

            app_lustamp = adjust_timestamp(latest.last_updated_timestamp, APP_TIMEZONE_CUTOFF)
            # app_man_lustamp = latest.last_manual_update_timestamp # no need to adjust this one
            app_man_lustamp = adjust_timestamp(latest.last_manual_update_timestamp, APP_TIMEZONE_CUTOFF)
            td = jcreated - app_lustamp
            mtd = jcreated - app_man_lustamp
            diff = td.total_seconds()
            mdiff = mtd.total_seconds()

            # was the journal created after the application by greater than the threshold?
            if diff > THRESHOLD:
                last_edit = ""
                last_accept = ""

                edit_query = deepcopy(PROV_QUERY)
                edit_query["query"]["bool"]["must"][0]["term"]["resource_id.exact"] = latest.id
                edit_query["query"]["bool"]["must"][1]["term"]["action.exact"] = "edit"
                provs = Provenance.q2obj(q=edit_query)
                if len(provs) > 0:
                    last_edit = provs[0].last_updated

                accept_query = deepcopy(PROV_QUERY)
                accept_query["query"]["bool"]["must"][0]["term"]["resource_id.exact"] = latest.id
                accept_query["query"]["bool"]["must"][1]["term"]["action.exact"] = "status:accepted"
                provs = Provenance.q2obj(q=accept_query)
                if len(provs) > 0:
                    last_accept = provs[0].last_updated

                out_applications.writerow([journal.id, journal.created_date, journal.last_update_request, latest.id, latest.last_updated, latest.application_status, diff, last_edit, last_accept])

            # was the journal (in doaj) created before the application by greater than the threshold, and is it in a state other than rejected
            if mdiff < -1 * THRESHOLD and latest.application_status != constants.APPLICATION_STATUS_REJECTED and journal.is_in_doaj():
                out_reapps.writerow([journal.id, journal.created_date, journal.last_update_request, latest.id, latest.created_date, latest.last_updated, latest.last_manual_update, latest.application_status, mdiff])

            # now figure out if the account is missing
            owner = journal.owner
            if owner is None:
                out_accounts.writerow([journal.id, journal.created_date, journal.last_update_request, str(journal.is_in_doaj()), "NO OWNER"])
            else:
                acc = Account.pull(owner)
                if acc is None:
                    out_accounts.writerow([journal.id, journal.created_date, journal.last_update_request, str(journal.is_in_doaj()), owner])

        print "processed", counter, "journals"
Example #29
0
    def test_01_reject_application(self,
                                   name,
                                   application,
                                   application_status,
                                   account,
                                   prov,
                                   current_journal,
                                   note,
                                   save,
                                   raises=None):

        #######################################
        ## set up

        if save == "fail":
            Suggestion.save = mock_save_fail

        ap = None
        journal = None
        if application == "exists":
            ap = Suggestion(
                **ApplicationFixtureFactory.make_application_source())
            ap.set_application_status(application_status)
            ap.set_id(ap.makeid())
            ap.remove_notes()
            if current_journal == "yes":
                journal = Journal(**JournalFixtureFactory.make_journal_source(
                    in_doaj=True))
                journal.set_id(journal.makeid())
                journal.set_current_application(ap.id)
                journal.save(blocking=True)
                ap.set_current_journal(journal.id)
            else:
                ap.remove_current_journal()

        acc = None
        if account == "publisher":
            acc = Account(**AccountFixtureFactory.make_publisher_source())
        elif account == "admin":
            acc = Account(
                **AccountFixtureFactory.make_managing_editor_source())

        provenance = None
        if prov != "none":
            provenance = prov == "true"

        thenote = None
        if note == "yes":
            thenote = "abcdefg"

        ########################################
        ## execute

        svc = DOAJ.applicationService()
        if raises is not None and raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.reject_application(ap, acc, provenance, note=thenote)
        else:
            svc.reject_application(ap, acc, provenance, note=thenote)
            time.sleep(1)

            #######################################
            ## Check

            ap2 = Suggestion.pull(ap.id)
            assert ap2 is not None
            assert ap2.application_status == constants.APPLICATION_STATUS_REJECTED
            assert ap2.current_journal is None

            # check the updated and manually updated date are essentially the same (they can theoretically differ
            # by a small amount just based on when they are set)
            updated_spread = abs(
                (ap2.last_updated_timestamp -
                 ap2.last_manual_update_timestamp).total_seconds())
            assert updated_spread <= 1.0

            if current_journal == "yes" and journal is not None:
                j2 = Journal.pull(journal.id)
                assert j2 is not None
                assert j2.current_application is None
                assert ap2.related_journal == j2.id

            if prov == "true":
                pr = Provenance.get_latest_by_resource_id(ap.id)
                assert pr is not None

            if note == "yes":
                assert len(ap2.notes) == 1
                assert ap2.notes[0].get("note") == "abcdefg"
            elif note == "no":
                assert len(ap2.notes) == 0
Example #30
0
    def test_02_application_2_journal(self, name, application_type, manual_update_arg, app_key_properties, current_journal, raises):
        # set up for the test
        #########################################

        cj = None
        has_seal = bool(randint(0, 1))
        application = None
        if application_type == "present":
            application = Suggestion(**ApplicationFixtureFactory.make_application_source())
            application.set_id(application.makeid())
            application.remove_contacts()
            application.remove_editor_group()
            application.remove_editor()
            application.remove_owner()
            application.remove_current_journal()
            application.remove_notes()

            if app_key_properties == "yes":
                application.add_contact("Application", "*****@*****.**")
                application.set_editor_group("appeditorgroup")
                application.set_editor("appeditor")
                application.set_owner("appowner")

            application.set_seal(has_seal)
            application.add_note("Application Note")

            if current_journal == "present":
                journal = Journal(**JournalFixtureFactory.make_journal_source())
                journal.remove_contacts()
                journal.add_contact("Journal", "*****@*****.**")
                journal.set_editor_group("journaleditorgroup")
                journal.set_editor("journaleditor")
                journal.set_owner("journalowner")
                journal.remove_current_application()
                journal.remove_notes()
                journal.add_note("Journal Note")
                journal.save(blocking=True)
                application.set_current_journal(journal.id)
                cj = journal
            elif current_journal == "missing":
                application.set_current_journal("123456789987654321")

        manual_update = None
        if manual_update_arg == "true":
            manual_update = True
        elif manual_update_arg == "false":
            manual_update = False

        # execute the test
        ########################################

        svc = DOAJ.applicationService()
        if raises is not None and raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.application_2_journal(application, manual_update)
        else:
            journal = svc.application_2_journal(application, manual_update)

            # check the result
            ######################################

            assert journal is not None
            assert isinstance(journal, Journal)
            assert journal.is_in_doaj() is True

            jbj = journal.bibjson().data
            del jbj["active"]
            assert jbj == application.bibjson().data

            if current_journal == "present":
                assert len(journal.related_applications) == 3
            else:
                assert len(journal.related_applications) == 1
            related = journal.related_application_record(application.id)
            assert related is not None

            if manual_update_arg == "true":
                assert journal.last_manual_update is not None and journal.last_manual_update != "1970-01-01T00:00:00Z"

            if app_key_properties == "yes":
                contacts = journal.contacts()
                assert len(contacts) == 1
                assert contacts[0].get("name") == "Application"
                assert contacts[0].get("email") == "*****@*****.**"
                assert journal.editor_group == "appeditorgroup"
                assert journal.editor == "appeditor"
                assert journal.owner == "appowner"
                assert journal.has_seal() == has_seal

                if current_journal == "present":
                    assert len(journal.notes) == 2
                else:
                    assert len(journal.notes) == 1

            elif app_key_properties == "no":
                if current_journal == "present":
                    contacts = journal.contacts()
                    assert len(contacts) == 1
                    assert contacts[0].get("name") == "Journal"
                    assert contacts[0].get("email") == "*****@*****.**"
                    assert journal.editor_group == "journaleditorgroup"
                    assert journal.editor == "journaleditor"
                    assert journal.owner == "journalowner"
                    assert journal.has_seal() == has_seal
                    assert len(journal.notes) == 2

                elif current_journal == "none" or current_journal == "missing":
                    contacts = journal.contacts()
                    assert len(contacts) == 0
                    assert journal.editor_group is None
                    assert journal.editor is None
                    assert journal.owner is None
                    assert journal.has_seal() == has_seal
                    assert len(journal.notes) == 1

            if current_journal == "present":
                assert cj.id == journal.id
                assert cj.created_date == journal.created_date
Example #31
0
conn = esprit.raw.Connection(source.get("host"), source.get("index"))

with open("application_2_journal.csv", "wb") as f:
    writer = csv.writer(f)
    writer.writerow([
        "application", "app_created", "app_last_update", "app_last_manual_update", "app_adjusted_lmu", "app_issns",
        "journal_matches", "journal", "journal_created", "journal_reapp", "journal_issns",
        "jc_ac_diff", "jc_lmua_diff", "mc1", "lra_ac_diff", "mc2",
        "is_match", "reason"
    ])

    # first, get each application and consider it
    counter = 0
    for result in esprit.tasks.scroll(conn, "suggestion", keepalive="1m"):
        counter += 1
        application = Suggestion(**result)
        application.remove_related_journal()

        # find all the journals that this application could be associated with (which we need to do by issn)
        issns = application.bibjson().issns()

        # query by each issn individually, because we're looking for the widest possible map.  Querying by
        # both would require both issns match
        related_journals = []
        related_journal_ids = []
        for issn in issns:
            journals = Journal.find_by_issn(issn)
            for journal in journals:
                if journal.id not in related_journal_ids:
                    related_journal_ids.append(journal.id)
                    related_journals.append(journal)
def applications_inconsistencies(outfile_later, outfile_missing, conn):
    with codecs.open(outfile_later, "wb",
                     "utf-8") as f, codecs.open(outfile_missing, "wb",
                                                "utf-8") as g:

        out_later = csv.writer(f)
        out_later.writerow([
            "Application ID", "Application Last Updated",
            "Latest Provenance Recorded", "Difference"
        ])

        out_missing = UnicodeWriter(g)
        out_missing.writerow([
            "Application ID", "Application Last Manual Update",
            "Latest Provenance Record", "ISSNs", "Title"
        ])

        counter = 0
        for result in esprit.tasks.scroll(conn, "suggestion", keepalive="45m"):
            counter += 1
            application = Suggestion(**result)
            print counter, application.id

            # Part 1 - later provenance records exist
            latest_prov = Provenance.get_latest_by_resource_id(application.id)
            if latest_prov is not None:
                lustamp = adjust_timestamp(application.last_updated_timestamp,
                                           APP_TIMEZONE_CUTOFF)
                created = latest_prov.created_date
                pstamp = latest_prov.created_timestamp
                td = pstamp - lustamp
                diff = td.total_seconds()

                if diff > THRESHOLD:
                    out_later.writerow([
                        application.id, application.last_updated, created, diff
                    ])

            # Part 2 - missing journals
            if application.application_status == constants.APPLICATION_STATUS_ACCEPTED:
                missing = False

                # find the matching journals by issn or by title
                matching_journals = Journal.find_by_issn(
                    application.bibjson().issns())
                if len(matching_journals) == 0:
                    # Have another go, find by title
                    matching_journals = Journal.find_by_title(
                        application.bibjson().title)

                # if there are no matching journals, it is missing.
                if len(matching_journals) == 0:
                    missing = True
                else:
                    # if there are matching journals, find out if any of them are in the doaj.  If none, then journal is still missing
                    those_in_doaj = len(
                        [j for j in matching_journals if j.is_in_doaj()])
                    if those_in_doaj == 0:
                        missing = True

                # if the journal is missing, record it
                if missing:
                    created = ""
                    if latest_prov is not None:
                        created = latest_prov.created_date
                    out_missing.writerow([
                        application.id, application.last_manual_update,
                        created, " ".join(application.bibjson().issns()),
                        application.bibjson().title
                    ])

        print "processed", counter, "suggestions"
Example #33
0
    def test_01_update_request(self, name, journal_id, journal_lock,
                               account, account_role, account_is_owner,
                               current_applications, application_lock, application_status,
                               completed_applications, raises,
                               return_app, return_jlock, return_alock,
                               db_jlock, db_alock, db_app):

        ###############################################
        ## set up

        # create the journal
        journal = None
        jid = None
        if journal_id == "valid":
            journal = Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True))
            journal.remove_related_applications()
            journal.remove_current_application()
            jid = journal.id
        elif journal_id == "not_in_doaj":
            journal = Journal(**JournalFixtureFactory.make_journal_source(in_doaj=False))
            journal.remove_related_applications()
            journal.remove_current_application()
            jid = journal.id
        elif journal_id == "missing":
            jid = uuid.uuid4().hex

        acc = None
        if account == "yes":
            acc = Account(**AccountFixtureFactory.make_publisher_source())
            if account_role == "none":
                acc.remove_role("publisher")
            elif account_role == "admin":
                acc.remove_role("publisher")
                acc.add_role("admin")
            acc.set_id(acc.makeid())
            if account_is_owner == "yes":
                acc.set_id(journal.owner)

        if journal_lock == "yes":
            lock.lock("journal", jid, "someoneelse", blocking=True)

        latest_app = None
        current_app_count = int(current_applications)
        for i in range(current_app_count):
            app = Suggestion(**ApplicationFixtureFactory.make_application_source())
            app.set_id(app.makeid())
            app.set_created("198" + str(i) + "-01-01T00:00:00Z")
            app.set_current_journal(jid)
            app.save()
            latest_app = app
            if journal is not None:
                journal.set_current_application(app.id)

        comp_app_count = int(completed_applications)
        for i in range(comp_app_count):
            app = Suggestion(**ApplicationFixtureFactory.make_application_source())
            app.set_id(app.makeid())
            app.set_created("197" + str(i) + "-01-01T00:00:00Z")
            app.set_related_journal(jid)
            app.save()
            if journal is not None:
                journal.add_related_application(app.id, date_accepted=app.created_date)

        if current_app_count == 0 and comp_app_count == 0:
            # save at least one record to initialise the index mapping, otherwise tests fail
            app = Suggestion(**ApplicationFixtureFactory.make_application_source())
            app.set_id(app.makeid())
            app.save()

        if application_lock == "yes":
            lock.lock("suggestion", latest_app.id, "someoneelse", blocking=True)

        if application_status != "n/a":
            latest_app.set_application_status(application_status)
            latest_app.save(blocking=True)

        # finally save the journal record, ensuring we get a blocking save, so everything
        # above here should be synchronised with the repo
        if journal is not None:
            journal.save(blocking=True)

        ###########################################################
        # Execution

        svc = DOAJ.applicationService()
        if raises != "":
            with self.assertRaises(EXCEPTIONS[raises]):
                svc.update_request_for_journal(jid, acc)
        else:
            application, jlock, alock = svc.update_request_for_journal(jid, acc)

            # we need to sleep, so the index catches up
            time.sleep(1)

            if return_app == "none":
                assert application is None
            elif return_app == "yes":
                assert application is not None

            if return_jlock == "none":
                assert jlock is None
            elif return_jlock == "yes":
                assert jlock is not None

            if return_alock == "none":
                assert alock is None
            elif return_alock == "yes":
                assert alock is not None

            if db_jlock == "no" and acc is not None:
                assert not lock.has_lock("journal", jid, acc.id)
            elif db_jlock == "yes" and acc is not None:
                assert lock.has_lock("journal", jid, acc.id)

            if db_alock == "no" and application.id is not None and acc is not None:
                assert not lock.has_lock("suggestion", application.id, acc.id)
            elif db_alock == "yes" and application.id is not None and acc is not None:
                assert lock.has_lock("suggestion", application.id, acc.id)

            if db_app == "no" and application.id is not None:
                indb = Suggestion.q2obj(q="id.exact:" + application.id)
                assert indb is None
            elif db_app == "yes" and application.id is not None:
                indb = Suggestion.q2obj(q="id.exact:" + application.id)
                assert indb is not None

            if current_app_count == 0 and comp_app_count == 0 and application is not None:
                assert application.article_metadata is None
                assert application.articles_last_year is None
            elif application is not None:
                assert application.article_metadata is not None
                assert application.articles_last_year is not None