def metadata(): # if this is a get request, give the blank form - there is no edit feature if request.method == "GET": form = ArticleForm() return render_template('publisher/metadata.html', form=form) # if this is a post request, a form button has been hit and we need to do # a bunch of work elif request.method == "POST": form = ArticleForm(request.form) # first we need to do any server-side form modifications which # the user might request by pressing the add/remove authors buttons more_authors = request.values.get("more_authors") remove_author = None for v in request.values.keys(): if v.startswith("remove_authors"): remove_author = v.split("-")[1] # if the user wants more authors, add an extra entry if more_authors: form.authors.append_entry() return render_template('publisher/metadata.html', form=form) # if the user wants to remove an author, do the various back-flips required if remove_author is not None: keep = [] while len(form.authors.entries) > 0: entry = form.authors.pop_entry() if entry.short_name == "authors-" + remove_author: break else: keep.append(entry) while len(keep) > 0: form.authors.append_entry(keep.pop().data) return render_template('publisher/metadata.html', form=form) # if we get to here, then this is the full submission, and we need to # validate and return enough_authors = _validate_authors(form) if form.validate(): # if the form validates, then we have to do our own bit of validation, # which is to check that there is at least one author supplied if not enough_authors: return render_template('publisher/metadata.html', form=form, author_error=True) else: xwalk = ArticleFormXWalk() art = xwalk.crosswalk_form(form) articleService = DOAJ.articleService() articleService.create_article( art, current_user._get_current_object()) flash("Article created/updated", "success") form = ArticleForm() return render_template('publisher/metadata.html', form=form) else: return render_template('publisher/metadata.html', form=form, author_error=not enough_authors)
def setUp(self): super(TestBLLArticleBatchCreateArticle, self).setUp() self.svc = DOAJ.articleService() self._is_legitimate_owner = self.svc.is_legitimate_owner self._get_duplicate = self.svc.get_duplicate self._issn_ownership_status = self.svc.issn_ownership_status self._get_journal = Article.get_journal
def setUp(self): super(TestBLLArticleBatchCreateArticle, self).setUp() self.svc = DOAJ.articleService() self._is_legitimate_owner = self.svc.is_legitimate_owner self._get_duplicate = self.svc.get_duplicate self._issn_ownership_status = self.svc.issn_ownership_status self._get_journal = Article.get_journal
def setUp(self): super(TestBLLPrepareUpdatePublisher, self).setUp() self.svc = DOAJ.articleService() self.is_id_updated = self.svc._doi_or_fulltext_updated self.has_permission = self.svc.has_permissions self.merge = Article.merge acc_source = AccountFixtureFactory.make_publisher_source() self.publisher = Account(**acc_source)
def setUp(self): super(TestBLLArticleCreateArticle, self).setUp() self.svc = DOAJ.articleService() self.is_legitimate_owner = self.svc.is_legitimate_owner self.ownership = self.svc.issn_ownership_status self.duplicate = self.svc.get_duplicate self.permission = self.svc.has_permissions self.prepare_update_admin = self.svc._prepare_update_admin self.prepare_update_publisher = self.svc._prepare_update_publisher
def create(cls, articles, account): # We run through the articles once, validating in dry-run mode # and deduplicating as we go. Then we .save() everything once # we know all incoming articles are valid. # as long as authentication (in the layer above) has been successful, and the account exists, then # we are good to proceed if account is None: raise Api401Error() # convert the data into a suitable article models articles = [ArticlesCrudApi.prep_article(data) for data in articles] articleService = DOAJ.articleService() try: result = articleService.batch_create_articles(articles, account) return [a.id for a in articles] except exceptions.IngestException as e: raise Api400Error(e.message)
def create(cls, articles, account): # We run through the articles once, validating in dry-run mode # and deduplicating as we go. Then we .save() everything once # we know all incoming articles are valid. # as long as authentication (in the layer above) has been successful, and the account exists, then # we are good to proceed if account is None: raise Api401Error() # convert the data into a suitable article models articles = [ArticlesCrudApi.prep_article(data) for data in articles] articleService = DOAJ.articleService() try: result = articleService.batch_create_articles(articles, account, add_journal_info=True) return [a.id for a in articles] except exceptions.IngestException as e: raise Api400Error(e.message)
def test_01_discover_duplicates(self, name, kwargs): article_arg = kwargs.get("article") owner_arg = kwargs.get("owner") article_doi_arg = kwargs.get("article_doi") doi_duplicate_arg = kwargs.get("doi_duplicate") article_fulltext_arg = kwargs.get("article_fulltext") fulltext_duplicate_arg = kwargs.get("fulltext_duplicate") articles_by_doi_arg = kwargs.get("articles_by_doi") articles_by_fulltext_arg = kwargs.get("articles_by_fulltext") raises_arg = kwargs.get("raises") raises = EXCEPTIONS.get(raises_arg) ############################################### ## set up owner = None if owner_arg != "none": owner = Account(**AccountFixtureFactory.make_publisher_source()) owner_id = None if owner is not None: owner_id = owner.id # create a journal for the owner if owner_arg not in ["none"]: source = JournalFixtureFactory.make_journal_source(in_doaj=True) journal = Journal(**source) journal.set_owner(owner.id) journal.bibjson().remove_identifiers() journal.bibjson().add_identifier("eissn", "1234-5678") journal.bibjson().add_identifier("pissn", "9876-5432") journal.save() # determine what we need to load into the index article_ids = [] aids_block = [] if owner_arg not in ["none", "no_articles"]: for i, ident in enumerate(IDENTS): the_doi = ident["doi"] if doi_duplicate_arg == "padded": the_doi = " " + the_doi + " " elif doi_duplicate_arg == "prefixed": the_doi = "https://dx.doi.org/" + the_doi the_fulltext = ident["fulltext"] if article_fulltext_arg != "invalid": if fulltext_duplicate_arg == "padded": the_fulltext = " http:" + the_fulltext elif fulltext_duplicate_arg == "http": the_fulltext = "http:" + the_fulltext elif fulltext_duplicate_arg == "https": the_fulltext = "https:" + the_fulltext else: the_fulltext = "http:" + the_fulltext source = ArticleFixtureFactory.make_article_source( eissn="1234-5678", pissn="9876-5432", doi=the_doi, fulltext=the_fulltext) article = Article(**source) article.set_id() article.save(blocking=True) article_ids.append(article.id) aids_block.append((article.id, article.last_updated)) # generate our incoming article article = None doi = None fulltext = None if article_arg == "yes": eissn = "1234=5678" # one matching pissn = "6789-1234" # the other not - issn matches are not relevant to this test if article_doi_arg in ["yes", "padded"]: doi = "10.1234/abc/11" if doi_duplicate_arg in ["yes", "padded"]: doi = IDENTS[0]["doi"] if article_doi_arg == "padded": doi = " doi:" + doi + " " elif article_doi_arg in ["invalid"]: doi = IDENTS[-1]["doi"] if article_fulltext_arg in ["yes", "padded", "https"]: fulltext = "//example.com/11" if fulltext_duplicate_arg in ["yes", "padded", "https"]: fulltext = IDENTS[0]["fulltext"] if fulltext_duplicate_arg == "padded": fulltext = " http:" + fulltext + " " elif fulltext_duplicate_arg == "https": fulltext = "https:" + fulltext else: fulltext = "http:" + fulltext elif article_fulltext_arg == "invalid": fulltext = IDENTS[-1]["fulltext"] source = ArticleFixtureFactory.make_article_source( eissn=eissn, pissn=pissn, doi=doi, fulltext=fulltext) article = Article(**source) # we need to do this if doi or fulltext are none, because the factory will set a default if we don't # provide them if doi is None: article.bibjson().remove_identifiers("doi") if fulltext is None: article.bibjson().remove_urls("fulltext") article.set_id() Article.blockall(aids_block) ########################################################### # Execution svc = DOAJ.articleService() if raises is not None: with self.assertRaises(raises): svc.discover_duplicates(article) else: possible_articles = svc.discover_duplicates(article) if articles_by_doi_arg == "yes": assert "doi" in possible_articles assert len(possible_articles["doi"]) == 1 # if this is the "invalid" doi, then we expect it to match the final article, otherwise match the first if article_doi_arg == "invalid": assert possible_articles["doi"][0].id == article_ids[-1] else: assert possible_articles["doi"][0].id == article_ids[0] else: if possible_articles is not None: assert "doi" not in possible_articles if articles_by_fulltext_arg == "yes": assert "fulltext" in possible_articles assert len(possible_articles["fulltext"]) == 1 # if this is the "invalid" fulltext url, then we expect it to match the final article, otherwise match the first if article_fulltext_arg == "invalid": assert possible_articles["fulltext"][0].id == article_ids[ -1] else: assert possible_articles["fulltext"][0].id == article_ids[ 0] else: if possible_articles is not None: assert "fulltext" not in possible_articles
def test_01_is_legitimate_owner(self, name, kwargs): article_arg = kwargs.get("article") owner_arg = kwargs.get("owner") article_eissn_arg = kwargs.get("article_eissn") article_pissn_arg = kwargs.get("article_pissn") seen_eissn_arg = kwargs.get("seen_eissn") seen_pissn_arg = kwargs.get("seen_pissn") journal_owner_arg = kwargs.get("journal_owner") raises_arg = kwargs.get("raises") legit_arg = kwargs.get("legit") raises = EXCEPTIONS.get(raises_arg) ############################################### ## set up owner = None if owner_arg != "none": owner = Account(**AccountFixtureFactory.make_publisher_source()) owner_id = None if owner is not None: owner_id = owner.id # generate our incoming article article = None eissn = None pissn = None if article_arg == "exists": source = ArticleFixtureFactory.make_article_source() article = Article(**source) article.set_id() article.bibjson().remove_identifiers("pissn") if article_pissn_arg == "yes": pissn = "1234-5678" article.bibjson().add_identifier("pissn", pissn) article.bibjson().remove_identifiers("eissn") if article_eissn_arg == "yes": eissn = "9876-5432" article.bibjson().add_identifier("eissn", eissn) # assemble the issns that will appear to be in the index. One that is irrelevant, and just # serves to be "noise" in the database, and the other that matches the spec required by # the test issns = [("1111-1111", "2222-2222")] if eissn is not None and pissn is not None and seen_eissn_arg == "yes" and seen_pissn_arg == "yes": issns.append((eissn, pissn)) if eissn is not None and seen_eissn_arg == "yes": issns.append((eissn, None)) if pissn is not None and seen_pissn_arg == "yes": issns.append((None, pissn)) owners = [] if journal_owner_arg == "none": owners = [None] elif journal_owner_arg == "correct" and owner_id is not None: owners = [owner_id] elif journal_owner_arg == "incorrect": owners = ["randomowner"] elif journal_owner_arg == "mix" and owner_id is not None: owners.append(owner_id) owners.append("randomowner") owners.append(None) mock = ModelJournalMockFactory.find_by_issn(issns, owners) Journal.find_by_issn = mock ########################################################### # Execution svc = DOAJ.articleService() if raises is not None: with self.assertRaises(raises): svc.is_legitimate_owner(article, owner_id) else: legit = svc.is_legitimate_owner(article, owner_id) if legit_arg == "no": assert legit is False elif legit_arg == "yes": assert legit is True
def setUp(self): super(TestBLLPrepareUpdatePublisher, self).setUp() self.svc = DOAJ.articleService() self.is_id_updated = self.svc._doi_or_fulltext_updated self.merge = Article.merge self.pull = Article.pull
def _process(self, file_upload): job = self.background_job upload_dir = app.config.get("UPLOAD_DIR") path = os.path.join(upload_dir, file_upload.local_filename) if not os.path.exists(path): job.add_audit_message( u"File not found at path {} . Retrying job later.".format( path)) count = self.get_param(job.params, "attempts") retry_limit = app.config.get("HUEY_TASKS", {}).get("ingest_articles", {}).get("retries", 0) self.set_param(job.params, "attempts", count + 1) if retry_limit <= count: job.add_audit_message( u"File still not found at path {} . Giving up.".format( path)) job.fail() raise RetryException() job.add_audit_message(u"Importing from {x}".format(x=path)) articleService = DOAJ.articleService() account = models.Account.pull(file_upload.owner) xwalk_name = app.config.get("ARTICLE_CROSSWALKS", {}).get(file_upload.schema) xwalk = plugin.load_class(xwalk_name)() ingest_exception = False result = {} try: with open(path) as handle: articles = xwalk.crosswalk_file( handle, add_journal_info=False ) # don't import the journal info, as we haven't validated ownership of the ISSNs in the article yet for article in articles: article.set_upload_id(file_upload.id) result = articleService.batch_create_articles( articles, account, add_journal_info=True) except IngestException as e: job.add_audit_message( u"IngestException: {msg}. Inner message: {inner}. Stack: {x}". format(msg=e.message, inner=e.inner_message, x=e.trace())) file_upload.failed(e.message, e.inner_message) result = e.result try: file_failed(path) ingest_exception = True except: job.add_audit_message( u"Error cleaning up file which caused IngestException: {x}" .format(x=traceback.format_exc())) except (DuplicateArticleException, ArticleNotAcceptable) as e: job.add_audit_message( u"One or more articles did not contain either a DOI or a Fulltext URL" ) file_upload.failed( u"One or more articles did not contain either a DOI or a Fulltext URL" ) try: file_failed(path) except: job.add_audit_message( u"Error cleaning up file which caused Exception: {x}". format(x=traceback.format_exc())) return except Exception as e: job.add_audit_message( u"Unanticipated error: {x}".format(x=traceback.format_exc())) file_upload.failed("Unanticipated error when importing articles") try: file_failed(path) except: job.add_audit_message( u"Error cleaning up file which caused Exception: {x}". format(x=traceback.format_exc())) return success = result.get("success", 0) fail = result.get("fail", 0) update = result.get("update", 0) new = result.get("new", 0) shared = result.get("shared", []) unowned = result.get("unowned", []) unmatched = result.get("unmatched", []) if success == 0 and fail > 0 and not ingest_exception: file_upload.failed("All articles in file failed to import") job.add_audit_message("All articles in file failed to import") if success > 0 and fail == 0: file_upload.processed(success, update, new) if success > 0 and fail > 0: file_upload.partial(success, fail, update, new) job.add_audit_message( "Some articles in file failed to import correctly, so no articles imported" ) file_upload.set_failure_reasons(list(shared), list(unowned), list(unmatched)) job.add_audit_message("Shared ISSNs: " + ", ".join(list(shared))) job.add_audit_message("Unowned ISSNs: " + ", ".join(list(unowned))) job.add_audit_message("Unmatched ISSNs: " + ", ".join(list(unmatched))) if not ingest_exception: try: os.remove(path) # just remove the file, no need to keep it except Exception as e: job.add_audit_message( u"Error while deleting file {x}: {y}".format(x=path, y=e.message))
def test_01_issn_ownership_status(self, name, kwargs): article_arg = kwargs.get("article") owner_arg = kwargs.get("owner") article_eissn_arg = kwargs.get("article_eissn") article_pissn_arg = kwargs.get("article_pissn") seen_eissn_arg = kwargs.get("seen_eissn") seen_pissn_arg = kwargs.get("seen_pissn") journal_owner_arg = kwargs.get("journal_owner") raises_arg = kwargs.get("raises") raises = EXCEPTIONS.get(raises_arg) ############################################### ## set up owner = None if owner_arg != "none": owner = Account(**AccountFixtureFactory.make_publisher_source()) owner_id = None if owner is not None: owner_id = owner.id # generate our incoming article article = None eissn = None pissn = None if article_arg == "exists": source = ArticleFixtureFactory.make_article_source() article = Article(**source) article.set_id() article.bibjson().remove_identifiers("pissn") if article_pissn_arg == "yes": pissn = "1234-5678" article.bibjson().add_identifier("pissn", pissn) article.bibjson().remove_identifiers("eissn") if article_eissn_arg == "yes": eissn = "9876-5432" article.bibjson().add_identifier("eissn", eissn) issns = [] if eissn is not None and pissn is not None and seen_eissn_arg == "yes" and seen_pissn_arg == "yes": issns.append((eissn, pissn)) if eissn is not None and seen_eissn_arg == "yes": issns.append((eissn, "4321-9876")) issns.append((eissn, None)) if pissn is not None and seen_pissn_arg == "yes": issns.append(("6789-4321", pissn)) issns.append((None, pissn)) owners = [] if journal_owner_arg == "none": owners = [None] elif journal_owner_arg == "correct" and owner_id is not None: owners = [owner_id] elif journal_owner_arg == "incorrect": owners = ["randomowner"] elif journal_owner_arg == "mix" and owner_id is not None: owners.append(owner_id) owners.append("randomowner") owners.append(None) mock = ModelJournalMockFactory.find_by_issn(issns, owners) Journal.find_by_issn = mock ########################################################### # Execution svc = DOAJ.articleService() if raises is not None: with self.assertRaises(raises): svc.issn_ownership_status(article, owner_id) else: owned, shared, unowned, unmatched = svc.issn_ownership_status( article, owner_id) owned_count = 0 if seen_eissn_arg == "yes" and eissn is not None and journal_owner_arg in [ "correct" ]: assert eissn in owned owned_count += 1 elif eissn is not None: assert eissn not in owned if seen_pissn_arg == "yes" and pissn is not None and journal_owner_arg in [ "correct" ]: assert pissn in owned owned_count += 1 elif pissn is not None: assert pissn not in owned assert len(owned) == owned_count shared_count = 0 if seen_eissn_arg == "yes" and eissn is not None and journal_owner_arg in [ "mix" ]: assert eissn in shared shared_count += 1 elif eissn is not None: assert eissn not in shared if seen_pissn_arg == "yes" and pissn is not None and journal_owner_arg in [ "mix" ]: assert pissn in shared shared_count += 1 elif pissn is not None: assert pissn not in shared assert len(shared) == shared_count unowned_count = 0 if seen_eissn_arg == "yes" and eissn is not None and journal_owner_arg in [ "incorrect", "none" ]: assert eissn in unowned unowned_count += 1 elif eissn is not None: assert eissn not in unowned if seen_pissn_arg == "yes" and pissn is not None and journal_owner_arg in [ "incorrect", "none" ]: assert pissn in unowned unowned_count += 1 elif pissn is not None: assert pissn not in unowned assert len(unowned) == unowned_count unmatched_count = 0 if seen_eissn_arg == "no" and eissn is not None: assert eissn in unmatched unmatched_count += 1 elif eissn is not None: assert eissn not in unmatched if seen_pissn_arg == "no" and pissn is not None: assert pissn in unmatched unmatched_count += 1 elif pissn is not None: assert pissn not in unmatched assert len(unmatched) == unmatched_count
def setUp(self): super(TestBLLArticleGetDuplicates, self).setUp() self.svc = DOAJ.articleService() self._old_discover_duplicates = self.svc.discover_duplicates
def test_01_discover_duplicates(self, name, kwargs): article_arg = kwargs.get("article") owner_arg = kwargs.get("owner") article_doi_arg = kwargs.get("article_doi") doi_duplicate_arg = kwargs.get("doi_duplicate") article_fulltext_arg = kwargs.get("article_fulltext") fulltext_duplicate_arg = kwargs.get("fulltext_duplicate") articles_by_doi_arg = kwargs.get("articles_by_doi") articles_by_fulltext_arg = kwargs.get("articles_by_fulltext") raises_arg = kwargs.get("raises") raises = EXCEPTIONS.get(raises_arg) ############################################### ## set up owner = None if owner_arg != "none": owner = Account(**AccountFixtureFactory.make_publisher_source()) owner_id = None if owner is not None: owner_id = owner.id # create a journal for the owner if owner_arg not in ["none"]: source = JournalFixtureFactory.make_journal_source(in_doaj=True) journal = Journal(**source) journal.set_owner(owner.id) journal.bibjson().remove_identifiers() journal.bibjson().add_identifier("eissn", "1234-5678") journal.bibjson().add_identifier("pissn", "9876-5432") journal.save(blocking=True) # determine what we need to load into the index article_ids = [] aids_block = [] if owner_arg not in ["none", "no_articles"]: for i, ident in enumerate(IDENTS): the_doi = ident["doi"] if doi_duplicate_arg == "padded": the_doi = " " + the_doi + " " elif doi_duplicate_arg == "prefixed": the_doi = "https://dx.doi.org/" + the_doi the_fulltext = ident["fulltext"] if article_fulltext_arg != "invalid": if fulltext_duplicate_arg == "padded": the_fulltext = " http:" + the_fulltext elif fulltext_duplicate_arg == "http": the_fulltext = "http:" + the_fulltext elif fulltext_duplicate_arg == "https": the_fulltext = "https:" + the_fulltext else: the_fulltext = "http:" + the_fulltext source = ArticleFixtureFactory.make_article_source(eissn="1234-5678", pissn="9876-5432", doi=the_doi, fulltext=the_fulltext) article = Article(**source) article.set_id() article.save() article_ids.append(article.id) aids_block.append((article.id, article.last_updated)) # generate our incoming article article = None doi = None fulltext = None if article_arg == "yes": eissn = "1234=5678" # one matching pissn = "6789-1234" # the other not - issn matches are not relevant to this test if article_doi_arg in ["yes", "padded"]: doi = "10.1234/abc/11" if doi_duplicate_arg in ["yes", "padded"]: doi = IDENTS[0]["doi"] if article_doi_arg == "padded": doi = " doi:" + doi + " " elif article_doi_arg in ["invalid"]: doi = IDENTS[-1]["doi"] if article_fulltext_arg in ["yes", "padded", "https"]: fulltext = "//example.com/11" if fulltext_duplicate_arg in ["yes", "padded", "https"]: fulltext = IDENTS[0]["fulltext"] if fulltext_duplicate_arg == "padded": fulltext = " http:" + fulltext + " " elif fulltext_duplicate_arg == "https": fulltext = "https:" + fulltext else: fulltext = "http:" + fulltext elif article_fulltext_arg == "invalid": fulltext = IDENTS[-1]["fulltext"] source = ArticleFixtureFactory.make_article_source(eissn=eissn, pissn=pissn, doi=doi, fulltext=fulltext) article = Article(**source) # we need to do this if doi or fulltext are none, because the factory will set a default if we don't # provide them if doi is None: article.bibjson().remove_identifiers("doi") if fulltext is None: article.bibjson().remove_urls("fulltext") article.set_id() Article.blockall(aids_block) ########################################################### # Execution svc = DOAJ.articleService() if raises is not None: with self.assertRaises(raises): svc.discover_duplicates(article, owner_id) else: possible_articles = svc.discover_duplicates(article, owner_id) if articles_by_doi_arg == "yes": assert "doi" in possible_articles assert len(possible_articles["doi"]) == 1 # if this is the "invalid" doi, then we expect it to match the final article, otherwise match the first if article_doi_arg == "invalid": assert possible_articles["doi"][0].id == article_ids[-1] else: assert possible_articles["doi"][0].id == article_ids[0] else: if possible_articles is not None: assert "doi" not in possible_articles if articles_by_fulltext_arg == "yes": assert "fulltext" in possible_articles assert len(possible_articles["fulltext"]) == 1 # if this is the "invalid" fulltext url, then we expect it to match the final article, otherwise match the first if article_fulltext_arg == "invalid": assert possible_articles["fulltext"][0].id == article_ids[-1] else: assert possible_articles["fulltext"][0].id == article_ids[0] else: if possible_articles is not None: assert "fulltext" not in possible_articles