def test_cmd1_bad_vals(self, mock):
"correct csv is still written despite bad location data"
# give article location a horrible value
horrible_very_bad_value = """'foo'o"ooo"o'"\"\';,oobarpants"""
self.ajson['article']['-meta']['location'] = horrible_very_bad_value
ajson_ingestor.ingest(self.ajson)
args = [self.nom, 'all-article-versions-as-csv']
retcode, stdout = self.call_command(*args)
self.assertEqual(retcode, 0)
rows = stdout.splitlines()
self.assertEqual(len(rows), 1) # 1 article, 1 row
row = rows[0]
bits = row.split(',')
# this naive split of a properly encoded csv-escaped value isn't going to work here
# self.assertEqual(len(bits), 3) # 3 bits to a row
# and consumers will probably choke on this bad data. ensure a warning is emitted
self.assertTrue(mock.warn.called_once())
# msid
self.assertEqual(int(bits[0]), self.msid)
# version
self.assertTrue(bits[1], self.version)
# location
self.assertTrue(bits[2], horrible_very_bad_value)
def test_article_ingest_data(self):
ajson_ingestor.ingest(self.ajson)
article_cases = [
('journal', logic.journal()),
('manuscript_id', 20105),
('volume', 5),
('doi', '10.7554/eLife.20105'),
('date_received', date(year=2016, month=7, day=27)),
('date_accepted', date(year=2016, month=10, day=3)),
]
art = models.Article.objects.get(manuscript_id=20105)
for attr, expected in article_cases:
actual = getattr(art, attr)
self.assertEqual(
actual, expected,
"expecting %r for %r got %r" % (expected, attr, actual))
article_version_cases = [
('article', art),
('title',
'An electrostatic selection mechanism controls sequential kinase signaling downstream of the T cell receptor'
), ('version', 1), ('status', 'poa'), ('datetime_published', None)
]
av = art.articleversion_set.all()[0]
for attr, expected in article_version_cases:
actual = getattr(av, attr)
self.assertEqual(
actual, expected,
"expecting %r for %r got %r" % (expected, attr, actual))
def test_ingest_events_no_history(self):
"when article xml has no accepted or received dates, events won't be created for them"
expected_events = [
models.DATETIME_ACTION_INGEST,
]
ajson_ingestor.ingest(self.without_history)
ael = models.ArticleEvent.objects.all()
self.assertEqual(ael.count(), len(expected_events))
def test_article_update_does_not_publish(self):
"ingesting article data twice still does not cause publication"
av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(av.datetime_published, None)
expected = "2016-04-13T01:00:00"
self.ajson['article']['published'] = expected
av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(av.datetime_published, None)
def test_article_update_does_not_publish(self):
"ingesting article data twice still does not cause publication"
_, _, av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(av.datetime_published, None)
expected = "2016-04-13T01:00:00"
self.ajson['article']['published'] = expected
_, _, av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(av.datetime_published, None)
def test_article_ingest(self):
"""valid article-json is successfully ingested, creating an article,
an article version and storing the ingestion request"""
self.assertEqual(models.Journal.objects.count(), 0)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.ArticleVersion.objects.count(), 0)
ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
def test_article_ingest(self):
"""valid article-json is successfully ingested, creating an article,
an article version and storing the ingestion request"""
self.assertEqual(models.Journal.objects.count(), 0)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.ArticleVersion.objects.count(), 0)
ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
def test_article_ingest_for_published_articles_succeeds_if_forced(self):
"ingesting article data for a published article version succeeds if force=True"
av = ajson_ingestor.ingest(self.ajson)
# edit data directly
av.datetime_published = '2001-01-01'
av.save()
self.assertTrue(av.published())
# attempt another ingest
expected_title = 'foo'
self.ajson['article']['title'] = expected_title
av = ajson_ingestor.ingest(self.ajson, force=True)
self.assertEqual(av.title, expected_title)
def test_article_ingest_for_published_articles_succeeds_if_forced(self):
"ingesting article data for a published article version succeeds if force=True"
_, _, av = ajson_ingestor.ingest(self.ajson)
# edit data directly
av.datetime_published = '2001-01-01'
av.save()
self.assertTrue(av.published())
# attempt another ingest
expected_title = 'foo'
self.ajson['article']['title'] = expected_title
_, _, av = ajson_ingestor.ingest(self.ajson, force=True)
self.assertEqual(av.title, expected_title)
def test_relations_replaced_during_ingest(self):
data = json.load(
open(
join(self.fixture_dir, 'relatedness',
'elife-13038-v1.xml.json')))
# point to 13620
data['article']['-related-articles-internal'] = ['13620']
ajson_ingestor.ingest(data, force=True)
avr = models.ArticleVersionRelation.objects.all()
self.assertEqual(
1, avr.count()) # still have just the one relationship ...
# and it's been updated
self.assertEqual(avr[0].related_to,
models.Article.objects.get(manuscript_id='13620'))
def test_v1_relations_preserved_ingesting_v2(self):
data = json.load(
open(
join(self.fixture_dir, 'relatedness',
'elife-13038-v1.xml.json')))
# point to 13620
data['article']['-related-articles-internal'] = ['13620']
data['article']['version'] = 2
ajson_ingestor.ingest(data)
avr = models.ArticleVersionRelation.objects.all().order_by(
'articleversion__version')
self.assertEqual(2, avr.count()) # two relations now
for i, avr in enumerate(avr):
self.assertEqual(avr.articleversion.version, i + 1)
def test_ingest_creates_events(self):
expected_events = [
models.DATE_XML_RECEIVED,
models.DATE_XML_ACCEPTED,
models.DATETIME_ACTION_INGEST,
]
ajson_ingestor.ingest(self.with_history)
ael = models.ArticleEvent.objects.all()
self.assertEqual(ael.count(), len(expected_events))
# order should be preserved
for event_type, event_obj in zip(expected_events, ael):
self.assertEqual(event_obj.event, event_type)
if event_type == models.DATETIME_ACTION_INGEST:
self.assertEqual(event_obj.value, "forced=False")
def test_article_publish_v2_forced(self):
"an unpublished v2 article can be successfully published again, if forced"
# ingest and publish the v1
av = ajson_ingestor.ingest(self.ajson)
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify and ingest+publish a v2
self.ajson['article']['version'] = 2
av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertTrue(av2.published())
# the v2 should have been published normally.
self.assertEqual(utils.ymd(datetime.now()),
utils.ymd(av2.datetime_published))
# give the article-json a 'versionDate' - this won't ordinarily happen until further down the line
# but lets embed this logic while it's still fresh in everybody's heads.
# modify the versionDate of the v2 and ingest+publish again
yesterday = datetime.now() - timedelta(days=1)
self.ajson['article']['versionDate'] = yesterday
av2v2 = ajson_ingestor.ingest_publish(self.ajson, force=True)
av2v2 = self.freshen(av2v2)
self.assertEqual(utils.ymd(yesterday),
utils.ymd(av2v2.datetime_published))
def test_article_publish_v1(self):
"an unpublished v1 article can be successfully published"
av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertFalse(av.published())
# publish
av = ajson_ingestor.publish(self.msid, self.version)
# aaand just make sure we still have the expected number of objects
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertTrue(av.published())
self.assertTrue(isinstance(av.datetime_published, datetime))
# the pubdate of an unpublished v1 article is the same as that found in the
# given json.
av = self.freshen(av)
expected_pubdate = utils.ymd(
utils.todt(self.ajson['article']['published']))
self.assertEqual(expected_pubdate, utils.ymd(av.datetime_published))
def test_article_json_not_stored_if_invalid(self):
"""INGEST and PUBLISH events cause the fragments to be merged and stored but
only if valid. ensure nothing is stored if result of merge is invalid"""
av = ajson_ingestor.ingest(self.invalid_ajson, force=True)
av = self.freshen(av)
self.assertEqual(av.article_json_v1, None)
self.assertEqual(av.article_json_v1_snippet, None)
def test_ingest_dry_run(self):
"specifying a dry run does not commit changes to database"
self.assertEqual(models.ArticleVersion.objects.count(), 0)
av = ajson_ingestor.ingest(self.ajson, dry_run=True)
self.assertEqual(models.ArticleVersion.objects.count(), 0)
self.assertEqual(av.version,
1) # all the data that would have been saved
def test_article_ingest_does_not_publish(self):
"""ingesting article json does not cause an article to become published
(gain a published date) even if a published date was supplied"""
expected = "2016-04-13T01:00:00"
self.ajson['article']['published'] = expected
_, _, av = ajson_ingestor.ingest(self.ajson)
self.assertNotEqual(av.datetime_published, expected)
def test_forced_ingest_passes_with_nonexistant_relations(self):
"an article that is related to an article that doesn't exist cannot be ingested (unless forced)."
models.ArticleVersionRelation.objects.all().delete()
data = json.load(
open(
join(self.fixture_dir, 'relatedness',
'elife-13038-v1.xml.json')))
data['article']['-related-articles-internal'] = ['42']
data['article']['version'] = 2
with self.settings(RELATED_ARTICLE_STUBS=False):
ajson_ingestor.ingest(data, force=True)
avr = models.ArticleVersionRelation.objects.all()
self.assertEqual(0, avr.count()) # not created ...
models.ArticleVersion.objects.get(article__manuscript_id=13038,
version=2) # ... but ingested
def test_article_ingest_update(self):
"ingesting article data twice successfully updates the Article object"
_, _, av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertEqual(av.title, self.ajson['article']['title'])
self.assertEqual(av.datetime_published, None) # not published
# do it again to cause an update
expected_title = 'flub'
self.ajson['article']['title'] = expected_title
_, _, av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertEqual(av.title, expected_title)
self.assertEqual(av.datetime_published, None) # still not published
def test_article_publish_v2_forced2(self):
"a PUBLISHED v2 article can be successfully published (again), if forced"
av = ajson_ingestor.ingest(self.ajson)
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify and ingest+publish a v2
self.ajson['article']['version'] = 2
# there is a versionDate here, but because we're not forcing it, it doesn't get looked for
# lax is the distributor of non-v1 pub dates. this may find their way into xml later, but
# they will always come from lax.
av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertTrue(av2.published())
self.assertEqual(utils.ymd(datetime.now()),
utils.ymd(av2.datetime_published))
# don't set a versionDate, just force a publish
# we expect the v2.datetime_publish to remain unchanged
del self.ajson['article'][
'versionDate'] # remember, this was copied from a v1 that had a versionDate!
av2v2 = ajson_ingestor.ingest_publish(self.ajson, force=True)
av2v2 = self.freshen(av2v2)
self.assertEqual(av2.datetime_published, av2v2.datetime_published)
def test_article_publish_v2_forced(self):
"an unpublished v2 article can be successfully published again, if forced"
# ingest and publish the v1
_, _, av = ajson_ingestor.ingest(self.ajson)
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify and ingest+publish a v2
self.ajson['article']['version'] = 2
_, _, av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertTrue(av2.published())
# the v2 should have been published normally.
self.assertEqual(utils.ymd(datetime.now()), utils.ymd(av2.datetime_published))
# give the article-json a 'versionDate' - this won't ordinarily happen until further down the line
# but lets embed this logic while it's still fresh in everybody's heads.
# modify the versionDate of the v2 and ingest+publish again
yesterday = datetime.now() - timedelta(days=1)
self.ajson['article']['versionDate'] = yesterday
_, _, av2v2 = ajson_ingestor.ingest_publish(self.ajson, force=True)
av2v2 = self.freshen(av2v2)
self.assertEqual(utils.ymd(yesterday), utils.ymd(av2v2.datetime_published))
def test_article_ingest_update(self):
"ingesting article data twice successfully updates the Article object"
av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertEqual(av.title, self.ajson['article']['title'])
self.assertEqual(av.datetime_published, None) # not published
# do it again to cause an update
expected_title = 'flub'
self.ajson['article']['title'] = expected_title
av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertEqual(av.title, expected_title)
self.assertEqual(av.datetime_published, None) # still not published
def test_article_ingest_does_not_publish(self):
"""ingesting article json does not cause an article to become published
(gain a published date) even if a published date was supplied"""
expected = "2016-04-13T01:00:00"
self.ajson['article']['published'] = expected
av = ajson_ingestor.ingest(self.ajson)
self.assertNotEqual(av.datetime_published, expected)
def test_article_json_not_stored_if_invalid(self):
"""INGEST and PUBLISH events cause the fragments to be merged and stored but
only if valid. ensure nothing is stored if result of merge is invalid"""
_, _, av = ajson_ingestor.ingest(self.invalid_ajson)
av = self.freshen(av)
self.assertEqual(av.article_json_v1, None)
self.assertEqual(av.article_json_v1_snippet, None)
def test_article_json_stored_if_valid(self):
"""INGEST and PUBLISH events cause the fragments to be merged and stored
but only if valid. ensure ajson is stored if result of merge is valid."""
_, _, av = ajson_ingestor.ingest(self.ajson)
av = self.freshen(av)
# not a great test ...
self.assertNotEqual(av.article_json_v1, None)
self.assertNotEqual(av.article_json_v1_snippet, None)
def test_related_events(self, notify_mock):
"aws_events.notify is called once for the article being ingested and once each for related articles"
ajson_ingestor.ingest(self.ajson1) # has 2 related
def event_msid(index):
args, first_arg = 1, 0
return notify_mock.mock_calls[index][args][first_arg]
# 10627 has two relations, 9561 and 9560
# ensure `notify` called once for each article
self.assertEqual(len(notify_mock.mock_calls), 3)
# ensure the events have the right manuscript id
# internal relationships, lowest to highest msid
self.assertEqual(event_msid(1), 9560)
self.assertEqual(event_msid(2), 9561)
def test_article_json_stored_if_valid(self):
"""INGEST and PUBLISH events cause the fragments to be merged and stored
but only if valid. ensure ajson is stored if result of merge is valid."""
av = ajson_ingestor.ingest(self.ajson)
av = self.freshen(av)
# not a great test ...
self.assertNotEqual(av.article_json_v1, None)
self.assertNotEqual(av.article_json_v1_snippet, None)
def test_article_publish_fails_if_already_published(self):
"a published article CANNOT be published again"
_, _, av = ajson_ingestor.ingest(self.ajson)
av = ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# publish again
self.assertRaises(StateError, ajson_ingestor.publish, self.msid, self.version)
def test_ingest_passes_and_creates_stubs_if_option_on(self):
"an article that is related to an article that doesn't exist will have the related Article created as a stub."
models.ArticleVersionRelation.objects.all().delete()
data = json.load(
open(
join(self.fixture_dir, 'relatedness',
'elife-13038-v1.xml.json')))
data['article']['-related-articles-internal'] = [42]
data['article']['version'] = 2
with self.settings(RELATED_ARTICLE_STUBS=True):
ajson_ingestor.ingest(data)
avr = models.ArticleVersionRelation.objects.all()
self.assertEqual(1, avr.count()) # relationship created ...
models.ArticleVersion.objects.get(article__manuscript_id=13038,
version=2) # ... and av ingested
models.Article.objects.get(
manuscript_id=42) # ... and stub created
def test_publish_dry_run(self):
"specifying a dry run does not commit changes to database"
_, _, saved_av = ajson_ingestor.ingest(self.ajson) # do an actual ingest first
unsaved_av = ajson_ingestor.publish(self.msid, self.version, dry_run=True)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
# ensure the article version stored has no published date
models.ArticleVersion.objects.get(pk=saved_av.pk, datetime_published=None)
# and that the object returned *does* have a datetime published
self.assertTrue(unsaved_av.published())
def test_article_ingest_fails_for_published_articles(self):
"ingesting article data for a published article version fails"
_, _, av = ajson_ingestor.ingest(self.ajson)
# edit data directly
av.datetime_published = '2001-01-01'
av.save()
self.assertTrue(av.published())
# attempt another ingest
self.assertRaises(StateError, ajson_ingestor.ingest, self.ajson)
def test_article_ingest_fails_for_published_articles(self):
"ingesting article data for a published article version fails"
av = ajson_ingestor.ingest(self.ajson)
# edit data directly
av.datetime_published = '2001-01-01'
av.save()
self.assertTrue(av.published())
# attempt another ingest
self.assertRaises(StateError, ajson_ingestor.ingest, self.ajson)
def test_article_publish_fails_if_already_published(self):
"a published article CANNOT be published again"
av = ajson_ingestor.ingest(self.ajson)
av = ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# publish again
self.assertRaises(StateError, ajson_ingestor.publish, self.msid,
self.version)
def test_related_events2(self):
"""aws_events.notify is called once for the article being ingested and once
each for related articles, including reverse relations"""
ajson_ingestor.ingest(self.ajson1) # has 2 related, 9561 and 9560
with patch('publisher.ajson_ingestor.aws_events.notify') as notify_mock:
def event_msid(index):
args, first_arg = 1, 0
return notify_mock.mock_calls[index][args][first_arg]
ajson_ingestor.ingest(self.ajson2) # has 2 related, 10627, 9561
self.assertEqual(len(notify_mock.mock_calls), 3)
# ensure the events have the right manuscript id
self.assertEqual(event_msid(0), 9560)
# internal relationships, lowest to highest msid
self.assertEqual(event_msid(1), 9561) # linked by 9560
self.assertEqual(event_msid(2), 10627) # links to 9560
def test_publish_dry_run(self):
"specifying a dry run does not commit changes to database"
saved_av = ajson_ingestor.ingest(
self.ajson) # do an actual ingest first
unsaved_av = ajson_ingestor.publish(self.msid,
self.version,
dry_run=True)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
# ensure the article version stored has no published date
models.ArticleVersion.objects.get(pk=saved_av.pk,
datetime_published=None)
# and that the object returned *does* have a datetime published
self.assertTrue(unsaved_av.published())
def handle_single(print_queue, action, infile, msid, version, force, dry_run):
data = None
log_context = {"msid": msid, "version": version}
LOG.info("attempting to %s article %s", action, msid, extra=log_context)
# read and check the article-json given, if necessary
try:
if action in [INGEST, BOTH]:
raw_data = infile.read()
log_context["data"] = str(raw_data[:25]) + "... (truncated)" if raw_data else ""
data = json.loads(raw_data)
# vagary of the CLI interface: article id and version are required
# these may not match the data given
data_version = data["article"].get("version")
if not data_version == version:
raise StateError(
"version in the data (%s) does not match version passed to script (%s)" % (data_version, version)
)
data_msid = int(data["article"]["id"])
if not data_msid == msid:
raise StateError(
"manuscript-id in the data (%s) does not match id passed to script (%s)" % (data_msid, msid)
)
except StateError as err:
error(print_queue, INVALID, err.message, log_context)
except ValueError as err:
msg = "could not decode the json you gave me: %r for data: %r" % (err.message, raw_data)
error(print_queue, INVALID, msg, log_context)
choices = {
# all these return a models.ArticleVersion object
INGEST: lambda msid, ver, force, data, dry: ajson_ingestor.ingest(data, force, dry_run=dry)[-1],
PUBLISH: lambda msid, ver, force, data, dry: ajson_ingestor.publish(msid, ver, force, dry_run=dry),
BOTH: lambda msid, ver, force, data, dry: ajson_ingestor.ingest_publish(data, force, dry_run=dry)[-1],
}
try:
av = choices[action](msid, version, force, data, dry_run)
success(print_queue, action, av, log_context, dry_run)
except StateError as err:
error(print_queue, INVALID, "failed to call action %r: %s" % (action, err.message), log_context)
except Exception as err:
msg = "unhandled exception attempting to %r article: %s" % (action, err)
LOG.exception(msg, extra=log_context)
error(print_queue, ERROR, msg, log_context)
def test_out_of_sequence_publish_fails(self):
"attempting to ingest an article with a version greater than another *published* version fails"
# ingest and publish a v1
av = ajson_ingestor.ingest(self.ajson) # v1
ajson_ingestor.publish(self.msid, self.version)
# now attempt to ingest a v3
self.ajson['article']['version'] = 3
self.assertRaises(StateError, ajson_ingestor.ingest, self.ajson)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
av = self.freshen(av)
self.assertEqual(av.version, 1) # assert the version hasn't changed
def test_out_of_sequence_publish_fails(self):
"attempting to ingest an article with a version greater than another *published* version fails"
# ingest and publish a v1
_, _, av = ajson_ingestor.ingest(self.ajson) # v1
ajson_ingestor.publish(self.msid, self.version)
# now attempt to ingest a v3
self.ajson['article']['version'] = 3
self.assertRaises(StateError, ajson_ingestor.ingest, self.ajson)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
av = self.freshen(av)
self.assertEqual(av.version, 1) # assert the version hasn't changed
def test_cmd1(self):
"""valid article-json is successfully ingested, creating an article,
an article version and storing the ingestion request"""
ajson_ingestor.ingest(self.ajson)
args = [self.nom, 'all-article-versions-as-csv']
retcode, stdout = self.call_command(*args)
self.assertEqual(retcode, 0)
rows = stdout.splitlines()
self.assertEqual(len(rows), 1) # 1 article, 1 row
row = rows[0]
bits = row.split(',')
self.assertEqual(len(bits), 3) # 3 bits to a row
# msid
self.assertEqual(int(bits[0]), self.msid)
# version
self.assertTrue(bits[1], self.version)
# location
expected_loc = "https://raw.githubusercontent.com/elifesciences/elife-article-xml/694f91de44ebc7cc61aba8be0982b7613cac8c3f/articles/elife-20105-v1.xml"
self.assertTrue(bits[2], expected_loc)
def test_out_of_sequence_ingest_fails2(self):
"attempting to ingest an article with a version greater than another unpublished version fails"
av = ajson_ingestor.ingest(self.ajson) # v1
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertEqual(av.version, 1)
# now attempt to ingest a v3
self.ajson['article']['version'] = 3
# force=True to get it past the validation errors.
# this used to work when we ignored validation errors on ingest
self.assertRaises(StateError, partial(ajson_ingestor.ingest,
force=True), self.ajson)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
av = self.freshen(av)
self.assertEqual(av.version, 1) # assert the version hasn't changed
def test_merge_ignores_unpublished_vor_when_setting_status_date(self):
"the first unpublished VOR doesn't get a value until it's published"
fixture = join(self.fixture_dir, 'ajson', 'elife-16695-v2.xml.json')
data = json.load(open(fixture, 'r'))
av2 = ajson_ingestor.ingest(data) # ingested, not published
av1 = self.freshen(self.av)
expected = '2016-08-16T00:00:00Z'
self.assertEqual(expected, av1.article_json_v1['statusDate'])
av2 = self.freshen(av2)
self.assertFalse(av2.datetime_published) # v2 is not published yet
self.assertTrue(av2.article_json_v1) # has article json attached
# v2 vor hasn't been published
self.assertEqual('preview', av2.article_json_v1['stage'])
self.assertFalse('statusDate' in av2.article_json_v1)
self.assertFalse('versionDate' in av2.article_json_v1)
def test_article_can_be_ingested_many_times_before_publication(self):
"before an article is published it can be ingested many times"
cases = json1, json2, json3 = map(copy.deepcopy, [self.ajson] * 3)
json2['article']['title'] = 'foo'
json3['article']['title'] = 'bar'
# iterate through the three different cases,
# assert each case is different from last
prev_fragment = None
for ajson in cases:
_, a, av = ajson_ingestor.ingest(ajson)
self.freshen(a)
fragment = a.articlefragment_set.get(type=models.XML2JSON)
if not prev_fragment:
prev_fragment = fragment
continue
self.assertNotEqual(prev_fragment.fragment, fragment.fragment)
def test_article_can_be_ingested_many_times_before_publication(self):
"before an article is published it can be ingested many times"
cases = json1, json2, json3 = lmap(copy.deepcopy, [self.ajson] * 3)
json2['article']['title'] = 'foo'
json3['article']['title'] = 'bar'
# iterate through the three different cases,
# assert each case is different from last
prev_fragment = None
for ajson in cases:
av = ajson_ingestor.ingest(ajson)
self.freshen(av)
fragment = av.article.articlefragment_set.get(type=models.XML2JSON)
if not prev_fragment:
prev_fragment = fragment
continue
self.assertNotEqual(prev_fragment.fragment, fragment.fragment)
def test_article_publish_succeeds_for_published_article_if_forced(self):
"publication of an already published article can occur only if forced"
_, _, av = ajson_ingestor.ingest(self.ajson)
av = ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
expected_pubdate = utils.ymd(utils.todt(self.ajson['article']['published']))
self.assertEqual(expected_pubdate, utils.ymd(av.datetime_published))
# publish again, no changes to pubdate expected
av = ajson_ingestor.publish(self.msid, self.version, force=True)
av = self.freshen(av)
self.assertEqual(expected_pubdate, utils.ymd(av.datetime_published))
# ingest new pubdate, force publication
new_pubdate = utils.todt('2016-01-01')
self.ajson['article']['published'] = new_pubdate
ajson_ingestor.ingest_publish(self.ajson, force=True)
av = self.freshen(av)
self.assertEqual(utils.ymd(new_pubdate), utils.ymd(av.datetime_published))
def test_article_publish_v2(self):
"an unpublished v2 article can be successfully published"
_, _, av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertFalse(av.published())
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify to a v2 and publish
self.ajson['article']['version'] = 2
_, _, av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertEqual(models.ArticleVersion.objects.count(), 2)
self.assertTrue(av2.published())
self.assertEqual(utils.ymd(datetime.now()), utils.ymd(av2.datetime_published))
def test_article_publish_succeeds_for_published_article_if_forced(self):
"publication of an already published article can occur only if forced"
av = ajson_ingestor.ingest(self.ajson)
av = ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
expected_pubdate = utils.ymd(
utils.todt(self.ajson['article']['published']))
self.assertEqual(expected_pubdate, utils.ymd(av.datetime_published))
# publish again, no changes to pubdate expected
av = ajson_ingestor.publish(self.msid, self.version, force=True)
av = self.freshen(av)
self.assertEqual(expected_pubdate, utils.ymd(av.datetime_published))
# ingest new pubdate, force publication
new_pubdate = utils.todt('2016-01-01')
self.ajson['article']['published'] = new_pubdate
ajson_ingestor.ingest_publish(self.ajson, force=True)
av = self.freshen(av)
self.assertEqual(utils.ymd(new_pubdate),
utils.ymd(av.datetime_published))
def test_article_publish_v2_forced2(self):
"a PUBLISHED v2 article can be successfully published (again), if forced"
_, _, av = ajson_ingestor.ingest(self.ajson)
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify and ingest+publish a v2
self.ajson['article']['version'] = 2
# there is a versionDate here, but because we're not forcing it, it doesn't get looked for
# lax is the distributor of non-v1 pub dates. this may find their way into xml later, but
# they will always come from lax.
_, _, av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertTrue(av2.published())
self.assertEqual(utils.ymd(datetime.now()), utils.ymd(av2.datetime_published))
# don't set a versionDate, just force a publish
# we expect the v2.datetime_publish to remain unchanged
del self.ajson['article']['versionDate'] # remember, this was copied from a v1 that had a versionDate!
_, _, av2v2 = ajson_ingestor.ingest_publish(self.ajson, force=True)
av2v2 = self.freshen(av2v2)
self.assertEqual(av2.datetime_published, av2v2.datetime_published)
def test_article_publish_v1(self):
"an unpublished v1 article can be successfully published"
_, _, av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertFalse(av.published())
# publish
av = ajson_ingestor.publish(self.msid, self.version)
# aaand just make sure we still have the expected number of objects
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertTrue(av.published())
self.assertTrue(isinstance(av.datetime_published, datetime))
# the pubdate of an unpublished v1 article is the same as that found in the
# given json.
av = self.freshen(av)
expected_pubdate = utils.ymd(utils.todt(self.ajson['article']['published']))
self.assertEqual(expected_pubdate, utils.ymd(av.datetime_published))
def test_ingest_dry_run(self):
"specifying a dry run does not commit changes to database"
self.assertEqual(models.ArticleVersion.objects.count(), 0)
_, _, av = ajson_ingestor.ingest(self.ajson, dry_run=True)
self.assertEqual(models.ArticleVersion.objects.count(), 0)
self.assertEqual(av.version, 1) # all the data that would have been saved