def test_response_types(self):
# ingest the poa and vor versions
for path in [self.ajson_fixture_v1, self.ajson_fixture_v2]:
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
# map the known types to expected types
art_list_type = 'application/vnd.elife.article-list+json;version=1'
art_poa_type = 'application/vnd.elife.article-poa+json;version=1'
art_vor_type = 'application/vnd.elife.article-vor+json;version=1'
art_history_type = 'application/vnd.elife.article-history+json;version=1'
case_list = {
reverse('v2:article-list'): art_list_type,
reverse('v2:article', kwargs={'id': self.msid}): art_vor_type,
reverse('v2:article-version-list', kwargs={'id': self.msid}): art_history_type,
reverse('v2:article-version', kwargs={'id': self.msid, 'version': 1}): art_poa_type,
reverse('v2:article-version', kwargs={'id': self.msid, 'version': 2}): art_vor_type,
}
# test
for url, expected_type in case_list.items():
resp = self.c.get(url)
self.assertEqual(resp.status_code, 200,
"url %r failed to complete: %s" % (url, resp.status_code))
self.assertEqual(resp.content_type, expected_type,
"%r failed to return %r: %s" % (url, expected_type, resp.content_type))
def setUp(self):
self.journal = logic.journal()
import_all = [
'00353.1', # discussion, VOR
'00385.1', # commentary, VOR
'01328.1', # correction, VOR
'02619.1', # editorial, VOR
'03401.1', # research, POA
'03401.2', # POA
'03401.3', # VOR
'03665.1', # research, VOR
'06250.1', # research, POA
'06250.2', # POA
'06250.3', # VOR
'07301.1', # research, VOR
'08025.1', # research, POA
'08025.2', # VOR
'09571.1', # research, POA
]
for subdir in import_all:
fname = subdir.replace('.', '-v')
fname = "elife-%s.xml.json" % fname
path = join(self.fixture_dir, 'ppp2', fname)
ajson_ingestor.ingest_publish(
self.load_ajson(path)) # strip relations
self.vor_version_count = 9
self.poa_version_count = 6
self.total_version_count = self.vor_version_count + self.poa_version_count
self.poa_art_count = 1
self.vor_art_count = 9
self.total_art_count = self.poa_art_count + self.vor_art_count
def setUp(self):
ingest_these = [
"elife-01968-v1.xml.json",
"elife-16695-v1.xml.json",
"elife-16695-v2.xml.json",
"elife-20125-v1.xml.json",
#"elife-16695-v3.xml.json"
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
data = self.load_ajson(join(ajson_dir, ingestable))
ajson_ingestor.ingest_publish(data)
self.msid1 = 1968
self.msid2 = 16695
self.msid3 = 20125
self.av = models.ArticleVersion.objects.get(
article__manuscript_id=self.msid1, version=1)
self.a = models.Article.objects.get(
manuscript_id=self.msid2) # note: no version information
self.citation = {
"type": "external-article",
"articleTitle":
"Transcriptional amplification in tumor cells with elevated c-Myc",
"journal": "Cell",
"authorLine": "C. Y. Lin et al",
"uri": "https://doi.org/10.1016/j.cell.2012.08.026"
}
self.citation2 = copy.deepcopy(self.citation)
self.citation2['uri'] = 'https://doi.org/10.0000/foo'
self.bad_citation = copy.deepcopy(self.citation)
self.bad_citation['uri'] = 'paaaants'
def setUp(self):
ingest_these = [
"elife-01968-v1.xml.json", # => 01749
"elife-16695-v1.xml.json", # => []
#"elife-16695-v2.xml.json",
"elife-20125-v1.xml.json", # poa
#"elife-16695-v3.xml.json"
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
path = join(ajson_dir, ingestable)
data = json.load(open(path, 'r'))
# remove these values here so they don't interfere in creation
utils.delall(
data,
['-related-articles-internal', '-related-articles-external'])
ajson_ingestor.ingest_publish(data)
self.msid1 = 1968
self.msid2 = 16695
self.msid3 = 20125
self.av = models.ArticleVersion.objects.get(
article__manuscript_id=self.msid1, version=1)
self.a = models.Article.objects.get(
manuscript_id=self.msid2) # note: no version information
def setUp(self):
ingest_these = [
#"elife-01968-v1.xml.json",
"dummyelife-20125-v1.xml.json", # poa
"dummyelife-20125-v2.xml.json", # poa
"dummyelife-20125-v3.xml.json", # vor
# NOT VALID, doesn't ingest
#"elife-16695-v1.xml.json",
#"elife-16695-v2.xml.json",
#"elife-16695-v3.xml.json", # vor
"dummyelife-20105-v1.xml.json", # poa
"dummyelife-20105-v2.xml.json", # poa
"dummyelife-20105-v3.xml.json" # poa
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
path = join(ajson_dir, ingestable)
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
self.msid1 = 20125
self.msid2 = 20105
av = models.ArticleVersion.objects.get(article__manuscript_id=self.msid1, version=3)
av.datetime_published = av.datetime_published + timedelta(days=1) # helps debug ordering
av.save()
self.c = Client()
def test_article_publish_v2_forced(self):
"an unpublished v2 article can be successfully published again, if forced"
# ingest and publish the v1
_, _, av = ajson_ingestor.ingest(self.ajson)
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify and ingest+publish a v2
self.ajson['article']['version'] = 2
_, _, av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertTrue(av2.published())
# the v2 should have been published normally.
self.assertEqual(utils.ymd(datetime.now()), utils.ymd(av2.datetime_published))
# give the article-json a 'versionDate' - this won't ordinarily happen until further down the line
# but lets embed this logic while it's still fresh in everybody's heads.
# modify the versionDate of the v2 and ingest+publish again
yesterday = datetime.now() - timedelta(days=1)
self.ajson['article']['versionDate'] = yesterday
_, _, av2v2 = ajson_ingestor.ingest_publish(self.ajson, force=True)
av2v2 = self.freshen(av2v2)
self.assertEqual(utils.ymd(yesterday), utils.ymd(av2v2.datetime_published))
def test_ingest_publish_no_force(self):
"attempting to do an update without force=True fails"
# ingest once
ajson_ingestor.ingest_publish(self.ajson)
# attempt second ingest
self.assertRaises(StateError, ajson_ingestor.ingest_publish,
self.ajson)
def test_add_fragment_multiple_versions(self):
path = join(self.fixture_dir, 'ajson', "elife-20105-v2.xml.json")
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
key = 'test-frag'
url = reverse('v2:article-fragment',
kwargs={
'art_id': self.msid,
'fragment_id': key
})
fragment = {'title': 'Electrostatic selection'}
resp = self.ac.post(url,
json.dumps(fragment),
content_type="application/json")
self.assertEqual(resp.status_code, 200)
# fragment is served into all article versions
article_url = reverse('v2:article-version-list',
kwargs={'id': self.msid})
resp = self.c.get(article_url)
data = utils.json_loads(resp.content)
self.assertEquals(len(data['versions']), 2)
self.assertEqual(data['versions'][0]['title'], fragment['title'])
self.assertEqual(data['versions'][1]['title'], fragment['title'])
def setUp(self):
ingest_these = [
#"elife-01968-v1.xml.json",
"elife-20125-v1.xml.json", # poa
"elife-20125-v2.xml.json", # poa
"elife-20125-v3.xml.json", # vor
#"elife-16695-v1.xml.json",
#"elife-16695-v2.xml.json",
#"elife-16695-v3.xml.json", # vor
"elife-20105-v1.xml.json", # poa
"elife-20105-v2.xml.json", # poa
"elife-20105-v3.xml.json" # poa
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
path = join(ajson_dir, ingestable)
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
self.msid1 = 20125
self.msid2 = 20105
av = models.ArticleVersion.objects.get(
article__manuscript_id=self.msid1, version=3)
av.datetime_published = av.datetime_published + timedelta(
days=1) # helps debug ordering: 20125 is published after 20105
av.save()
self.c = Client()
self.ac = Client(**{
mware.CGROUPS: 'admin',
})
def test_article_publish_v2_forced(self):
"an unpublished v2 article can be successfully published again, if forced"
# ingest and publish the v1
av = ajson_ingestor.ingest(self.ajson)
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify and ingest+publish a v2
self.ajson['article']['version'] = 2
av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertTrue(av2.published())
# the v2 should have been published normally.
self.assertEqual(utils.ymd(datetime.now()),
utils.ymd(av2.datetime_published))
# give the article-json a 'versionDate' - this won't ordinarily happen until further down the line
# but lets embed this logic while it's still fresh in everybody's heads.
# modify the versionDate of the v2 and ingest+publish again
yesterday = datetime.now() - timedelta(days=1)
self.ajson['article']['versionDate'] = yesterday
av2v2 = ajson_ingestor.ingest_publish(self.ajson, force=True)
av2v2 = self.freshen(av2v2)
self.assertEqual(utils.ymd(yesterday),
utils.ymd(av2v2.datetime_published))
def test_article_publish_v2_forced2(self):
"a PUBLISHED v2 article can be successfully published (again), if forced"
av = ajson_ingestor.ingest(self.ajson)
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify and ingest+publish a v2
self.ajson['article']['version'] = 2
# there is a versionDate here, but because we're not forcing it, it doesn't get looked for
# lax is the distributor of non-v1 pub dates. this may find their way into xml later, but
# they will always come from lax.
av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertTrue(av2.published())
self.assertEqual(utils.ymd(datetime.now()),
utils.ymd(av2.datetime_published))
# don't set a versionDate, just force a publish
# we expect the v2.datetime_publish to remain unchanged
del self.ajson['article'][
'versionDate'] # remember, this was copied from a v1 that had a versionDate!
av2v2 = ajson_ingestor.ingest_publish(self.ajson, force=True)
av2v2 = self.freshen(av2v2)
self.assertEqual(av2.datetime_published, av2v2.datetime_published)
def test_ingest_publish_force(self):
"we can do silent corrections/updates if we force it to"
av = ajson_ingestor.ingest_publish(self.ajson)
expected_title = 'pants-party'
self.ajson['article']['title'] = expected_title
av = ajson_ingestor.ingest_publish(self.ajson, force=True)
av = self.freshen(av)
self.assertEqual(av.title, expected_title)
def test_ingest_publish_force(self):
"we can do silent corrections/updates if we force it to"
_, _, av = ajson_ingestor.ingest_publish(self.ajson)
expected_title = 'pants-party'
self.ajson['article']['title'] = expected_title
_, _, av = ajson_ingestor.ingest_publish(self.ajson, force=True)
av = self.freshen(av)
self.assertEqual(av.title, expected_title)
def test_publish_fails_if_invalid(self):
"an article cannot be published if it's article-json is invalid."
self.ajson['article']['title'] = ''
# ValidationError is raised during ingest
try:
ajson_ingestor.ingest_publish(self.ajson)
except StateError as err:
self.assertEqual(err.code, codes.INVALID)
self.assertEqual(err.trace.strip()[:15], "'' is too short")
def setUp(self):
ingest_these = [
"elife-16695-v1.xml.json", # research article
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
path = join(ajson_dir, ingestable)
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
self.msid = 16695
def setUp(self):
ingest_these = [
"elife-04718-v1.xml.json", # reverse relation to 13038
"elife-13038-v1.xml.json", # int relation to 04718
"elife-13620-v1.xml.json", # ext relation
]
ajson_dir = join(self.fixture_dir, 'relatedness')
for ingestable in ingest_these:
data = json.load(open(join(ajson_dir, ingestable), 'r'))
ajson_ingestor.ingest_publish(data)
self.assertEqual(models.ArticleVersion.objects.count(), 3)
def setUp(self):
ingest_these = [
"elife-01968-v1.xml.json", "elife-16695-v1.xml.json",
"elife-16695-v2.xml.json", "elife-16695-v3.xml.json"
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
path = join(ajson_dir, ingestable)
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
self.msid1 = 1968
self.msid2 = 16695
def setUp(self):
path = join(self.fixture_dir, 'ajson', "dummyelife-20105-v1.xml.json")
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
self.msid = 20105
self.version = 1
self.av = models.ArticleVersion.objects.filter(article__manuscript_id=self.msid)[0]
self.assertTrue(self.av.published())
self.assertTrue(fragments.merge_if_valid(self.av))
self.c = Client()
def test_validate_forced_action(self):
"a dry ingest can be sent with a force flag on existing article to test validity of a silent correction"
# article exists
ajson = json.load(open(self.ajson_fixture_v1, 'r'))
ajson_ingestor.ingest_publish(ajson)
# a silent correction happens without problems
args = [
self.nom, '--ingest', '--dry-run', '--id', self.msid, '--version',
1, '--force', self.ajson_fixture_v1
]
errcode, stdout = self.call_command(*args)
self.assertEqual(errcode, 0)
def test_unacceptable_types(self):
ajson_ingestor.ingest_publish(json.load(open(self.ajson_fixture_v1, 'r')))
cases = [
# vor v2 or v3
"application/vnd.elife.article-vor+json; version=2, application/vnd.elife.article-vor+json; version=3",
# poa v2
"application/vnd.elife.article-poa+json; version=2",
# ??
"application/foo.bar.baz; version=1"
]
for header in cases:
resp = self.c.get(reverse('v2:article-version', kwargs={'id': self.msid, 'version': 1}), HTTP_ACCEPT=header)
self.assertEqual(resp.status_code, 406, "failed on case %r, got: %s" % (header, resp.status_code))
def test_validate_non_existant_av(self):
"a dry ingest can be done for a non-existant article version"
ajson = json.load(open(self.ajson_fixture_v1, 'r'))
ajson_ingestor.ingest_publish(ajson) # *v1*
self.assertEqual(models.ArticleVersion.objects.count(), 1)
args = [
self.nom, '--ingest', '--dry-run', '--id', self.msid, '--version',
2, self.ajson_fixture_v2
] # *v2*
errcode, stdout = self.call_command(*args)
self.assertEqual(errcode, 0)
self.assertEqual(models.ArticleVersion.objects.count(),
1) # still just one
def setUp(self):
ingest_these = [
"elife-01968-v1.xml.json",
"elife-16695-v1.xml.json",
"elife-16695-v2.xml.json",
"elife-16695-v3.xml.json"
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
path = join(ajson_dir, ingestable)
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
self.msid1 = 1968
self.msid2 = 16695
def test_accept_types(self):
"various accept headers return expected response"
ajson_ingestor.ingest_publish(json.load(open(self.ajson_fixture_v1, 'r')))
cases = [
"*/*",
"application/vnd.elife.article-poa+json; version=1, application/vnd.elife.article-vor+json; version=1",
"application/vnd.elife.article-poa+json; version=1",
"application/vnd.elife.article-vor+json; version=1", # yes, even though the returned result is a poa
# vor v1 or v2
"application/vnd.elife.article-vor+json; version=1, application/vnd.elife.article-vor+json; version=2",
]
for header in cases:
resp = self.c.get(reverse('v2:article-version', kwargs={'id': self.msid, 'version': 1}), HTTP_ACCEPT=header)
self.assertEqual(resp.status_code, 200)
def test_validate_without_force_flag(self):
"an invalid dry ingest fails obviously"
# article exists
ajson = json.load(open(self.ajson_fixture_v1, 'r'))
ajson_ingestor.ingest_publish(ajson)
# attempt validation of given data
args = [
self.nom, '--ingest', '--dry-run', '--id', self.msid, '--version',
1, self.ajson_fixture_v1
] # no force flag present
errcode, stdout = self.call_command(*args)
self.assertEqual(errcode, 1) # 1 = error
# error resp has a bucket of info
resp = json.loads(stdout)
self.assertTrue(
utils.has_all_keys(resp, ['code', 'message', 'comment', 'trace']))
def test_publish_creates_events(self):
expected_events = [
models.DATE_XML_RECEIVED,
models.DATE_XML_ACCEPTED,
models.DATETIME_ACTION_INGEST,
models.DATETIME_ACTION_PUBLISH,
]
ajson_ingestor.ingest_publish(self.with_history)
ael = models.ArticleEvent.objects.all()
self.assertEqual(ael.count(), len(expected_events))
# order should be preserved
for event_type, event_obj in zip(expected_events, ael):
self.assertEqual(event_obj.event, event_type)
if event_type == models.DATETIME_ACTION_PUBLISH:
self.assertEqual(event_obj.value, "forced=False")
def test_article_versions_list_placeholder(self):
"invalid article-json causes a placeholder to be served instead"
invalid_ajson = json.load(
open(join(self.fixture_dir, 'ajson', 'elife-20125-v4.xml.json'),
'r'))
invalid_ajson['article']['title'] = ''
av = ajson_ingestor.ingest_publish(invalid_ajson, force=True)
self.freshen(av)
# we now have a published article in lax with invalid article-json
resp = self.c.get(
reverse('v2:article-version-list', kwargs={'id': self.msid1}))
data = utils.json_loads(resp.content)
# the invalid-but-published culprit
v4 = data['versions'][-1]
expected_struct = utils.json_loads(
utils.json_dumps({
'-invalid': True,
'id': av.article.manuscript_id,
'status': av.status,
'published': av.article.datetime_published,
'version': 4,
'versionDate': av.datetime_published
}))
self.assertEqual(expected_struct, v4)
def setUp(self):
path = join(self.fixture_dir, 'ajson', "elife-20105-v1.xml.json")
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
self.msid = 20105
self.version = 1
self.av = models.ArticleVersion.objects.filter(
article__manuscript_id=self.msid)[0]
self.assertTrue(self.av.published())
self.assertTrue(fragments.merge_if_valid(self.av))
self.c = Client()
self.ac = Client(**{
mware.CGROUPS: 'admin',
})
def add_or_update_article(self, **adata):
"creates article+article-version stubs for testing"
replacements = [
('pub-date', 'published'),
('update', 'versionDate'),
]
renkeys(adata, replacements)
struct = {
'id':
utils.doi2msid(adata['doi'])
if 'doi' in adata else adata['manuscript_id'],
'volume':
1,
'type':
'research-article',
'title':
'[default]',
'version':
1,
'status':
models.VOR,
'published':
'2012-01-01T00:00:00Z'
}
struct.update(adata)
delall(struct, ['journal']) # can't be serialized, not utilised anyway
with self.settings(VALIDATE_FAILS_FORCE=False):
# bad ajson won't fail ingest
av = ajson_ingestor.ingest_publish({'article': struct}, force=True)
av.datetime_published = utils.todt(struct['published'])
av.save()
return av
def test_logic_doesnt_mangle_unicode(self):
"the api logic doesn't alter unicode values"
ajson = json.load(open(self.ajson_fixture1, 'r'))
ajson_ingestor.ingest_publish(ajson)
expected = ajson['snippet']['authors'][1]['name']['preferred']
# /articles/{id}
given = logic.most_recent_article_version(
self.msid).article_json_v1['authors'][1]['name']['preferred']
self.assertEqual(expected, given)
# /articles/{id}/versions/{version}
given = logic.article_version(
self.msid,
self.version).article_json_v1['authors'][1]['name']['preferred']
self.assertEqual(expected, given)
def test_ingest_publish_dont_alter_unicode(self):
"the unicode value in the scraped json isn't altered when it's ingested and published"
ajson = json.load(open(self.ajson_fixture1, 'r'))
av = ajson_ingestor.ingest_publish(ajson)
av = utils.freshen(av)
expected = ajson['snippet']['authors'][1]['name']['preferred']
given = av.article_json_v1['authors'][1]['name']['preferred']
self.assertEqual(expected, given)
def setUp(self):
# poa, published 2016-08-16T00:00:00Z
self.ajson_fixture = join(self.fixture_dir, 'ajson',
'elife-16695-v1.xml.json')
self.ajson = json.load(open(self.ajson_fixture, 'r'))
self.msid = self.ajson['article']['id']
self.version = self.ajson['article']['version'] # v1
self.av = ajson_ingestor.ingest_publish(self.ajson)
def setUp(self):
ingest_these = [
"elife-01968-v1.xml.json",
"elife-16695-v1.xml.json",
"elife-20125-v1.xml.json",
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
data = self.load_ajson(join(ajson_dir, ingestable))
ajson_ingestor.ingest_publish(data)
self.msid1 = 1968
self.msid2 = 16695
self.msid3 = 20125
self.av = models.ArticleVersion.objects.get(
article__manuscript_id=self.msid1, version=1)
def setUp(self):
self.nom = 'check_article'
self.msid_list = ["20125", "20105", "19264"] # 17267 17850 19083 18638 20143 21734 16578".split()
ingest_these = [
"dummyelife-20125-v1.xml.json", # poa
"dummyelife-20125-v2.xml.json", # poa
"dummyelife-20125-v3.xml.json", # vor
"dummyelife-20105-v1.xml.json", # poa
"dummyelife-20105-v2.xml.json", # poa
"dummyelife-20105-v3.xml.json" # poa, UNPUBLISHED
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
path = join(ajson_dir, ingestable)
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
def test_revalidate_article_from_cli(self):
"the revalidate command can be called from the CLI for all versions of an article"
# we need to ingest some more versions here
fixtures = sorted(
glob.glob(
join(self.fixture_dir, 'ajson',
'elife-%s-v*.xml.json' % self.msid2)))
for fixture in fixtures[1:]: # don't reingest v1
ajson_ingestor.ingest_publish(json.load(open(fixture, 'r')))
args = [self.nom, '--id', self.msid2]
errcode, stdout = self.call_command(*args)
self.assertEqual(errcode, 0)
report = json.loads(stdout)
self.assertEqual(
report['total-reset'],
3) # msid2 is untouched, all are valid, all should be reset
def test_response_types(self):
# ingest the poa and vor versions
for path in [self.ajson_fixture_v1, self.ajson_fixture_v2]:
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
# map the known types to expected types
art_list_type = 'application/vnd.elife.article-list+json;version=1'
art_poa_type = 'application/vnd.elife.article-poa+json;version=1'
art_vor_type = 'application/vnd.elife.article-vor+json;version=1'
art_history_type = 'application/vnd.elife.article-history+json;version=1'
art_related_type = 'application/vnd.elife.article-related+json;version=1'
case_list = {
reverse('v2:article-list'):
art_list_type,
reverse('v2:article', kwargs={'id': self.msid}):
art_vor_type,
reverse('v2:article-version-list', kwargs={'id': self.msid}):
art_history_type,
reverse('v2:article-version',
kwargs={
'id': self.msid,
'version': 1
}):
art_poa_type,
reverse('v2:article-version',
kwargs={
'id': self.msid,
'version': 2
}):
art_vor_type,
reverse('v2:article-relations', kwargs={'id': self.msid}):
art_related_type,
}
# test
for url, expected_type in case_list.items():
resp = self.c.get(url)
self.assertEqual(
resp.status_code, 200,
"url %r failed to complete: %s" % (url, resp.status_code))
self.assertEqual(
resp.content_type, expected_type,
"%r failed to return %r: %s" %
(url, expected_type, resp.content_type))
def setUp(self):
self.ajson_fixture = join(self.fixture_dir, 'ajson',
'elife-01968-v1.xml.json')
self.ajson = json.load(open(self.ajson_fixture, 'r'))
self.msid = self.ajson['article']['id']
self.version = self.ajson['article']['version'] # v1
# populate with an article. CREATES A FRAGMENT
self.av = ajson_ingestor.ingest_publish(self.ajson)
def setUp(self):
self.journal = publogic.journal()
import_all = [
'00353.1', # discussion, VOR
'00385.1', # commentary, VOR
'01328.1', # correction, VOR
'02619.1', # editorial, VOR
'03401.1', # research, POA
'03401.2', # POA
'03401.3', # VOR
'03665.1', # research, VOR
'06250.1', # research, POA
'06250.2', # POA
'06250.3', # VOR
'07301.1', # research, VOR
'08025.1', # research, POA
'08025.2', # VOR
'09571.1', # research, POA
]
for subdir in import_all:
fname = subdir.replace('.', '-v')
fname = "elife-%s.xml.json" % fname
path = join(self.fixture_dir, 'ppp2', fname)
ajson_ingestor.ingest_publish(
self.load_ajson(path)) # strip relations
# we need to coerce the data of the non-v1 articles a little
# as we removed the eif ingestor that bypassed business logic
cases = [
# vor
(3401, 3, "2014-08-01"),
(8025, 2, "2015-06-16"),
]
for msid, ver, dtstr in cases:
av = models.ArticleVersion.objects.get(article__manuscript_id=msid,
version=ver)
av.datetime_published = utils.todt(dtstr)
av.save()
self.vor_version_count = 9
self.poa_version_count = 6
self.poa_art_count = 1
self.vor_art_count = 9
def setUp(self):
self.nom = 'revalidate'
# publish four valid articles, ensure each is in different states
fixture_list = [
'01968', # SET
'16695', # RESET
'20105', # UNSET
'20125', # NOTSET
]
self.msid1, self.msid2, self.msid3, self.msid4 = fixture_list
self.version = '1'
for msid in fixture_list:
fixture_path = join(self.fixture_dir, 'ajson',
'elife-%s-v1.xml.json' % msid)
ajson_ingestor.ingest_publish(json.load(open(fixture_path, 'r')))
# SET
# remove the valid ajson so it gets set
av = article_logic.article_version(self.msid1, self.version)
av.article_json_v1 = av.article_json_v1_snippet = None
av.save()
# RESET
# already valid, nothing to do
# UNSET
# ensure the fragment is now invalid
av = article_logic.article_version(self.msid3, self.version)
frag = logic.get(av, models.XML2JSON)
frag.fragment['title'] = ''
frag.save()
# NOTSET
# remove the valid article-json
# ensure fragment is invalid and won't be set
av = article_logic.article_version(self.msid4, self.version)
av.article_json_v1 = av.article_json_v1_snippet = None
av.save()
frag = logic.get(av, models.XML2JSON)
frag.fragment['title'] = ''
frag.save()
def test_article_publish_succeeds_for_published_article_if_forced(self):
"publication of an already published article can occur only if forced"
_, _, av = ajson_ingestor.ingest(self.ajson)
av = ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
expected_pubdate = utils.ymd(utils.todt(self.ajson['article']['published']))
self.assertEqual(expected_pubdate, utils.ymd(av.datetime_published))
# publish again, no changes to pubdate expected
av = ajson_ingestor.publish(self.msid, self.version, force=True)
av = self.freshen(av)
self.assertEqual(expected_pubdate, utils.ymd(av.datetime_published))
# ingest new pubdate, force publication
new_pubdate = utils.todt('2016-01-01')
self.ajson['article']['published'] = new_pubdate
ajson_ingestor.ingest_publish(self.ajson, force=True)
av = self.freshen(av)
self.assertEqual(utils.ymd(new_pubdate), utils.ymd(av.datetime_published))
def test_unacceptable_types(self):
ajson_ingestor.ingest_publish(
json.load(open(self.ajson_fixture_v1, 'r')))
cases = [
# vor v2 or v3
"application/vnd.elife.article-vor+json; version=3, application/vnd.elife.article-vor+json; version=4",
# poa v2
"application/vnd.elife.article-poa+json; version=3",
# ??
"application/foo.bar.baz; version=1"
]
for header in cases:
resp = self.c.get(reverse('v2:article-version',
kwargs={
'id': self.msid,
'version': 1
}),
HTTP_ACCEPT=header)
self.assertEqual(
resp.status_code, 406,
"failed on case %r, got: %s" % (header, resp.status_code))
def handle_single(print_queue, action, infile, msid, version, force, dry_run):
data = None
log_context = {"msid": msid, "version": version}
LOG.info("attempting to %s article %s", action, msid, extra=log_context)
# read and check the article-json given, if necessary
try:
if action in [INGEST, BOTH]:
raw_data = infile.read()
log_context["data"] = str(raw_data[:25]) + "... (truncated)" if raw_data else ""
data = json.loads(raw_data)
# vagary of the CLI interface: article id and version are required
# these may not match the data given
data_version = data["article"].get("version")
if not data_version == version:
raise StateError(
"version in the data (%s) does not match version passed to script (%s)" % (data_version, version)
)
data_msid = int(data["article"]["id"])
if not data_msid == msid:
raise StateError(
"manuscript-id in the data (%s) does not match id passed to script (%s)" % (data_msid, msid)
)
except StateError as err:
error(print_queue, INVALID, err.message, log_context)
except ValueError as err:
msg = "could not decode the json you gave me: %r for data: %r" % (err.message, raw_data)
error(print_queue, INVALID, msg, log_context)
choices = {
# all these return a models.ArticleVersion object
INGEST: lambda msid, ver, force, data, dry: ajson_ingestor.ingest(data, force, dry_run=dry)[-1],
PUBLISH: lambda msid, ver, force, data, dry: ajson_ingestor.publish(msid, ver, force, dry_run=dry),
BOTH: lambda msid, ver, force, data, dry: ajson_ingestor.ingest_publish(data, force, dry_run=dry)[-1],
}
try:
av = choices[action](msid, version, force, data, dry_run)
success(print_queue, action, av, log_context, dry_run)
except StateError as err:
error(print_queue, INVALID, "failed to call action %r: %s" % (action, err.message), log_context)
except Exception as err:
msg = "unhandled exception attempting to %r article: %s" % (action, err)
LOG.exception(msg, extra=log_context)
error(print_queue, ERROR, msg, log_context)
def setUp(self):
ingest_these = [
#"elife-01968-v1.xml.json",
"dummyelife-20125-v1.xml.json", # poa
"dummyelife-20125-v2.xml.json", # poa
"dummyelife-20125-v3.xml.json", # vor
# NOT VALID, doesn't ingest
#"elife-16695-v1.xml.json",
#"elife-16695-v2.xml.json",
#"elife-16695-v3.xml.json", # vor
"dummyelife-20105-v1.xml.json", # poa
"dummyelife-20105-v2.xml.json", # poa
"dummyelife-20105-v3.xml.json" # poa, UNPUBLISHED
]
ajson_dir = join(self.fixture_dir, 'ajson')
for ingestable in ingest_these:
path = join(ajson_dir, ingestable)
ajson_ingestor.ingest_publish(json.load(open(path, 'r')))
self.msid1 = 20125
self.msid2 = 20105
# 2 article, 6 versions, 5 published, 1 unpublished
self.unpublish(self.msid2, version=3)
# an unauthenticated client
self.c = Client()
# an authenticated client
self.ac = Client(**{
'REMOTE_ADDR': '10.0.2.6',
mware.CGROUPS: 'user',
mware.CID: str(uuid.uuid4()),
mware.CUSER: '******'
})
def test_article_publish_v2_forced2(self):
"a PUBLISHED v2 article can be successfully published (again), if forced"
_, _, av = ajson_ingestor.ingest(self.ajson)
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify and ingest+publish a v2
self.ajson['article']['version'] = 2
# there is a versionDate here, but because we're not forcing it, it doesn't get looked for
# lax is the distributor of non-v1 pub dates. this may find their way into xml later, but
# they will always come from lax.
_, _, av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertTrue(av2.published())
self.assertEqual(utils.ymd(datetime.now()), utils.ymd(av2.datetime_published))
# don't set a versionDate, just force a publish
# we expect the v2.datetime_publish to remain unchanged
del self.ajson['article']['versionDate'] # remember, this was copied from a v1 that had a versionDate!
_, _, av2v2 = ajson_ingestor.ingest_publish(self.ajson, force=True)
av2v2 = self.freshen(av2v2)
self.assertEqual(av2.datetime_published, av2v2.datetime_published)
def test_ingest_publish(self):
"ensure the shortcut ingest_publish behaves as expected"
self.assertEqual(models.Journal.objects.count(), 0)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.ArticleVersion.objects.count(), 0)
_, _, av = ajson_ingestor.ingest_publish(self.ajson)
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
av = self.freshen(av)
self.assertEqual(av.version, 1)
self.assertTrue(av.published())
def test_ingest_publish_dry_run(self):
"specifying a dry run does not commit changes to database"
self.assertEqual(models.Journal.objects.count(), 0)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.ArticleVersion.objects.count(), 0)
j, a, av = ajson_ingestor.ingest_publish(self.ajson, dry_run=True)
# all counts are still zero
self.assertEqual(models.Journal.objects.count(), 0)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.ArticleVersion.objects.count(), 0)
# article version believes itself to be published
self.assertTrue(av.published())
def test_article_publish_v2(self):
"an unpublished v2 article can be successfully published"
_, _, av = ajson_ingestor.ingest(self.ajson)
self.assertEqual(models.Journal.objects.count(), 1)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.ArticleVersion.objects.count(), 1)
self.assertFalse(av.published())
ajson_ingestor.publish(self.msid, self.version)
av = self.freshen(av)
self.assertTrue(av.published())
# modify to a v2 and publish
self.ajson['article']['version'] = 2
_, _, av2 = ajson_ingestor.ingest_publish(self.ajson)
av2 = self.freshen(av2)
self.assertEqual(models.ArticleVersion.objects.count(), 2)
self.assertTrue(av2.published())
self.assertEqual(utils.ymd(datetime.now()), utils.ymd(av2.datetime_published))
def test_article_versions_list_placeholder(self):
invalid_ajson = json.load(open(join(self.fixture_dir, 'ajson', 'elife-20125-v4.xml.json'), 'r'))
invalid_ajson['article']['title'] = ''
_, _, av = ajson_ingestor.ingest_publish(invalid_ajson, force=True)
self.freshen(av)
# we now have a published article in lax with invalid article-json
resp = self.c.get(reverse('v2:article-version-list', kwargs={'id': self.msid1}))
data = json.loads(resp.content)
# the invalid-but-published culprit
v4 = data['versions'][-1]
expected_struct = json.loads(utils.json_dumps({
'-invalid': True,
'status': av.status,
'published': av.article.datetime_published,
'version': 4,
'versionDate': av.datetime_published
}))
self.assertEqual(expected_struct, v4)
def test_ingest_publish_no_force(self):
"attempting to do an update without force=True fails"
# ingest once
_, _, av = ajson_ingestor.ingest_publish(self.ajson)
# attempt second ingest
self.assertRaises(StateError, ajson_ingestor.ingest_publish, self.ajson)