def setUp(self):
self.c = Client()
self.journal = logic.journal()
an_hour_ago = utils.utcnow() - timedelta(hours=1)
many_hours_ago = an_hour_ago - timedelta(hours=999)
fmt = utils.ymdhms
self.article_data_list = [
{'title': 'foo',
'status': 'vor',
'version': 1,
'doi': "10.7554/eLife.00001",
'journal': self.journal,
'pub-date': fmt(an_hour_ago),
},
{'title': 'bar',
'status': 'vor',
'version': 1,
'doi': "10.7554/eLife.00002",
'journal': self.journal,
'pub-date': fmt(many_hours_ago),
},
{'title': 'baz',
'version': 1,
'status': 'poa', # **
'doi': "10.7554/eLife.00003",
'journal': self.journal,
'pub-date': fmt(an_hour_ago),
}
]
[logic.add_or_update_article(**article_data) for article_data in self.article_data_list]
def setUp(self):
self.c = Client()
an_hour_ago = utils.utcnow() - timedelta(hours=1)
many_hours_ago = an_hour_ago - timedelta(hours=999)
fmt = utils.ymdhms
self.article_data_list = [
{'title': 'foo',
'status': 'vor',
'version': 1,
'doi': "10.7554/eLife.00001",
'pub-date': fmt(an_hour_ago),
},
{'title': 'bar',
'status': 'vor',
'version': 1,
'doi': "10.7554/eLife.00002",
'pub-date': fmt(many_hours_ago),
},
{'title': 'baz',
'version': 1,
'status': 'poa', # **
'doi': "10.7554/eLife.00003",
'pub-date': fmt(an_hour_ago),
}
]
[self.add_or_update_article(**article_data) for article_data in self.article_data_list]
def add(art, event, value=None, datetime_event=None):
utils.ensure(art, "need art")
datetime_event = datetime_event or utils.utcnow()
struct = {
'event': event,
'value': str(value),
'datetime_event': datetime_event
}
create = update = True
ae, created, updated = \
create_or_update(models.ArticleEvent, struct, ['article', 'event', 'datetime_event'], create, update, article=art)
return ae
def test_ingest_from_cli(self):
"ingest script requires the --ingest flag and a source of data"
args = [self.nom, '--ingest', '--id', self.msid, '--version', self.version, self.ajson_fixture1]
errcode, stdout = self.call_command(*args)
self.assertEqual(errcode, 0)
# article has been ingested
self.assertEqual(models.ArticleVersion.objects.count(), 1)
# message returned is json encoded with all the right keys and values
result = json.loads(stdout.getvalue())
self.assertTrue(utils.has_all_keys(result, ['status', 'id', 'datetime']))
self.assertEqual(result['status'], 'ingested')
# the date and time is roughly the same as right now, ignoring microseconds
expected_datetime = utils.ymdhms(utils.utcnow())
self.assertEqual(result['datetime'][:20], expected_datetime[:20])
self.assertEqual(result['datetime'][-6:], expected_datetime[-6:])
def test_ingest_from_cli(self):
"ingest script requires the --ingest flag and a source of data"
args = [
self.nom, '--ingest', '--id', self.msid, '--version', self.version,
self.ajson_fixture1
]
errcode, stdout = self.call_command(*args)
self.assertEqual(errcode, 0)
# article has been ingested
self.assertEqual(models.ArticleVersion.objects.count(), 1)
# message returned is json encoded with all the right keys and values
result = json.loads(stdout)
self.assertTrue(
utils.has_all_keys(result, ['status', 'id', 'datetime']))
self.assertEqual(result['status'], 'ingested')
# the date and time is roughly the same as right now, ignoring microseconds
expected_datetime = utils.utcnow()
actual_datetime = utils.todt(result['datetime'])
delta = expected_datetime - actual_datetime
threshold = 2 # seconds
self.assertTrue(delta.seconds <= threshold)
def test_utcnow(self):
"utcnow returns a UTC datetime"
# TODO: this test could be improved
now = utils.utcnow()
self.assertEqual(now.tzinfo, pytz.utc)
def test_only_most_recent_article_versions_returned(self):
an_hour_ago = utils.utcnow() - timedelta(hours=1)
many_hours_ago = an_hour_ago - timedelta(hours=999)
fmt = utils.ymdhms
article_data_list = [
{'title': 'foo',
'version': 1,
'doi': "10.7554/eLife.00001",
'pub-date': fmt(an_hour_ago),
},
{'title': 'bar',
'version': 1,
'doi': "10.7554/eLife.00002",
'pub-date': fmt(many_hours_ago),
},
{'title': 'bar',
'version': 2,
'doi': "10.7554/eLife.00002",
'pub-date': fmt(many_hours_ago - timedelta(hours=1)),
'update': fmt(many_hours_ago - timedelta(hours=1)),
},
{'title': 'bar',
'version': 3,
'doi': "10.7554/eLife.00002",
'pub-date': fmt(many_hours_ago - timedelta(hours=2)),
'update': fmt(many_hours_ago - timedelta(hours=2)),
},
{'title': 'baz',
'version': 1,
'doi': "10.7554/eLife.00003",
'pub-date': fmt(an_hour_ago + timedelta(minutes=5)),
},
{'title': 'baz',
'version': 2,
'doi': "10.7554/eLife.00003",
'pub-date': fmt(an_hour_ago + timedelta(minutes=10)),
'update': fmt(an_hour_ago + timedelta(minutes=10)),
}
]
[self.add_or_update_article(**article_data) for article_data in article_data_list]
self.assertEqual(models.Article.objects.count(), 3)
self.assertEqual(models.ArticleVersion.objects.count(), 6)
total, avlist = logic.latest_article_version_list()
self.assertEqual(total, 3)
self.assertEqual(len(avlist), 3)
expected_version_order = [
('10.7554/eLife.00003', 2), # published less than an hour ago
('10.7554/eLife.00001', 1), # published an hour ago
('10.7554/eLife.00002', 3), # published many hours ago
]
for av, expected in zip(avlist, expected_version_order):
self.assertEqual(av.article.doi, expected[0])
self.assertEqual(av.version, expected[1])
def _publish(msid, version, force=False):
"""attach a `datetime_published` value to an article version. if none provided, use RIGHT NOW.
you cannot publish an already published article version unless force==True"""
try:
av = models.ArticleVersion.objects.get(article__manuscript_id=msid, version=version)
if av.published():
if not force:
raise StateError("refusing to publish an already published article version")
# NOTE: we don't use any other article fragments for determining the publication date
# except the xml->json fragment.
raw_data = fragments.get(av, XML2JSON)
# the json *will always* have a published date if v1 ...
if version == 1:
# pull that published date from the stored (but unpublished) article-json
# and set the pub-date on the ArticleVersion object
datetime_published = utils.todt(raw_data.get('published'))
if not datetime_published:
raise StateError("found 'published' value in article-json, but it's either null or unparsable as a datetime")
else:
# but *not* if it's > v1. in this case, we generate one.
if av.published() and force:
# this article version is already published and a force publish request has been sent
if False and 'versionDate' in raw_data: # fail this case for now.
# FUTURE CASE: when a 'versionDate' value is present in the article-json, use that.
# as of 2016-10-21 version history IS NOT captured in the xml,
# it won't be parsed by the bot-lax-adaptor and it
# won't find it's way here. this is a future-case only.
datetime_published = utils.todt(raw_data['versionDate'])
if not datetime_published:
raise StateError("found 'versionDate' value in article-json, but it's either null or unparseable as a datetime")
else:
# CURRENT CASE
# preserve the existing pubdate set by lax. ignore anything given in the ajson.
# if the pubdate for an article is to change, it must come from the xml (see above case)
datetime_published = av.datetime_published
else:
# CURRENT CASE
# this article version hasn't been published yet. use a value of RIGHT NOW as the published date.
datetime_published = utils.utcnow()
av.datetime_published = datetime_published
av.save()
# merge the fragments we have available and make them available for serving
# allow errors when the publish operation is being forced
fragments.merge_if_valid(av, quiet=force)
# notify event bus that article change has occurred
transaction.on_commit(partial(events.notify, av.article))
return av
except ValidationError:
raise StateError("refusing to publish an article '%sv%s' with invalid article-json" % (msid, version))
except models.ArticleFragment.DoesNotExist:
raise StateError("no 'xml->json' fragment found. being strict and failing this publish. please INGEST!")
except models.ArticleVersion.DoesNotExist:
# attempted to publish an article that doesn't exist ...
raise StateError("refusing to publish an article '%sv%s' that doesn't exist" % (msid, version))
def _publish(msid, version, force=False) -> models.ArticleVersion:
"""attach a `datetime_published` value to an article version. if none provided, use RIGHT NOW.
you cannot publish an already published article version unless force==True"""
try:
av = models.ArticleVersion.objects.get(article__manuscript_id=msid,
version=version)
if av.published():
if not force:
raise StateError(
codes.ALREADY_PUBLISHED,
"refusing to publish an already published article version")
# NOTE: we don't use any other article fragments for determining the publication date
# except the xml->json fragment.
raw_data = fragments.get(av, XML2JSON).fragment
# the json *will always* have a published date if v1 ...
if version == 1:
# pull that published date from the stored (but unpublished) article-json
# and set the pub-date on the ArticleVersion object
datetime_published = utils.todt(raw_data.get('published'))
if not datetime_published:
raise StateError(
codes.PARSE_ERROR,
"found 'published' value in article-json, but it's either null or unparsable as a date+time"
)
else:
# but *not* if it's > v1. in this case, we generate one.
if av.published() and force:
# this article version is already published and a force publish request has been sent
if False and 'versionDate' in raw_data: # fail this case for now.
# FUTURE CASE: when a 'versionDate' value is present in the article-json, use that.
# as of 2016-10-21 version history IS NOT captured in the xml,
# it won't be parsed by the bot-lax-adaptor and it
# won't find it's way here. this is a future-case only.
datetime_published = utils.todt(raw_data['versionDate'])
if not datetime_published:
raise StateError(
codes.PARSE_ERROR,
"found 'versionDate' value in article-json, but it's either null or unparseable as a datetime"
)
else:
# CURRENT CASE
# preserve the existing pubdate set by lax. ignore anything given in the ajson.
# if the pubdate for an article is to change, it must come from the xml (see above case)
datetime_published = av.datetime_published
else:
# CURRENT CASE
# this article version hasn't been published yet. use a value of RIGHT NOW as the published date.
datetime_published = utils.utcnow()
av.datetime_published = datetime_published
av.save()
events.ajson_publish_events(av, force)
# merge the fragments we have available and make them available for serving.
# allow errors when the publish operation is being forced.
fragments.set_article_json(
av, quiet=False if settings.VALIDATE_FAILS_FORCE else force)
# notify event bus that article change has occurred
transaction.on_commit(partial(aws_events.notify_all, av))
return av
except ValidationError as err:
# the problem isn't that the ajson is invalid, it's that we've allowed invalid ajson into the system
raise StateError(
codes.INVALID,
"refusing to publish an article '%sv%s' with invalid article-json: %s"
% (msid, version, err), err)
except models.ArticleFragment.DoesNotExist:
raise StateError(
codes.NO_RECORD,
"no 'xml->json' fragment found. being strict and failing this publish. please INGEST!"
)
except models.ArticleVersion.DoesNotExist:
# attempted to publish an article that doesn't exist ...
raise StateError(
codes.NO_RECORD,
"refusing to publish an article '%sv%s' that doesn't exist" %
(msid, version))
def test_only_most_recent_article_versions_returned(self):
an_hour_ago = utils.utcnow() - timedelta(hours=1)
many_hours_ago = an_hour_ago - timedelta(hours=999)
fmt = utils.ymdhms
article_data_list = [
{'title': 'foo',
'version': 1,
'doi': "10.7554/eLife.00001",
'pub-date': fmt(an_hour_ago),
},
{'title': 'bar',
'version': 1,
'doi': "10.7554/eLife.00002",
'pub-date': fmt(many_hours_ago),
},
{'title': 'bar',
'version': 2,
'doi': "10.7554/eLife.00002",
'pub-date': fmt(many_hours_ago - timedelta(hours=1)),
'update': fmt(many_hours_ago - timedelta(hours=1)),
},
{'title': 'bar',
'version': 3,
'doi': "10.7554/eLife.00002",
'pub-date': fmt(many_hours_ago - timedelta(hours=2)),
'update': fmt(many_hours_ago - timedelta(hours=2)),
},
{'title': 'baz',
'version': 1,
'doi': "10.7554/eLife.00003",
'pub-date': fmt(an_hour_ago + timedelta(minutes=5)),
},
{'title': 'baz',
'version': 2,
'doi': "10.7554/eLife.00003",
'pub-date': fmt(an_hour_ago + timedelta(minutes=10)),
'update': fmt(an_hour_ago + timedelta(minutes=10)),
}
]
[logic.add_or_update_article(**article_data) for article_data in article_data_list]
self.assertEqual(models.Article.objects.count(), 3)
self.assertEqual(models.ArticleVersion.objects.count(), 6)
avlist = logic.latest_article_versions()
self.assertEqual(len(avlist), 3)
expected_version_order = [
('10.7554/eLife.00003', 2), # published less than an hour ago
('10.7554/eLife.00001', 1), # published an hour ago
('10.7554/eLife.00002', 3), # published many hours ago
]
for av, expected in zip(avlist, expected_version_order):
self.assertEqual(av.article.doi, expected[0])
self.assertEqual(av.version, expected[1])
