def test_get_n_articles(self):
from amcat.tools import amcattest
codingjob1 = amcattest.create_test_job(10)
codingjob2 = amcattest.create_test_job(5)
jobs = CodingJob.objects.filter(id__in=[codingjob1.id, codingjob2.id])
s = self._get_serializer(jobs)
self.assertEqual(10, s.get_n_articles(codingjob1))
self.assertEqual(5, s.get_n_articles(codingjob2))
def test_get_n_articles(self):
from amcat.tools import amcattest
codingjob1 = amcattest.create_test_job(10)
codingjob2 = amcattest.create_test_job(5)
jobs = CodingJob.objects.filter(id__in=[codingjob1.id, codingjob2.id])
s = self._get_serializer(jobs)
self.assertEqual(10, s.get_n_articles(codingjob1))
self.assertEqual(5, s.get_n_articles(codingjob2))
def test_delete_setinuse(self):
"""Delete a job whose set is in use somewhere else"""
from amcat.models import ArticleSet
s = amcattest.create_test_set(articles=5)
j = amcattest.create_test_job(articleset=s)
j2 = amcattest.create_test_job(articleset=s)# use same article set
DeleteCodingJob(job=j.id).run()
self.assertFalse(CodingJob.objects.filter(pk=j.id).exists())
self.assertEquals(j2.articleset, s)
self.assertTrue(ArticleSet.objects.filter(pk=s.id).exists())
def test_delete_setinuse(self):
"""Delete a job whose set is in use somewhere else"""
from amcat.models import ArticleSet
s = amcattest.create_test_set(articles=5)
j = amcattest.create_test_job(articleset=s)
j2 = amcattest.create_test_job(articleset=s) # use same articleset
DeleteCodingJob(job=j.id).run()
self.assertFalse(CodingJob.objects.filter(pk=j.id).exists())
self.assertEquals(j2.articleset, s)
self.assertTrue(ArticleSet.objects.filter(pk=s.id).exists())
def test_api(self):
from amcat.models import CodingStatus
cj = amcattest.create_test_job()
# Test empty codingjob
res = self.get(CodingJobResource, id=cj.id)['results'][0]
self.assertTrue("n_codings_done" in res)
self.assertTrue("n_articles" in res)
self.assertEquals(1, res["n_articles"])
self.assertEquals(0, res["n_codings_done"])
# Add two codings
cj.codings.add(amcattest.create_test_coding(),
amcattest.create_test_coding())
res = self.get(CodingJobResource, id=cj.id)['results'][0]
self.assertEquals(1, res["n_articles"])
self.assertEquals(0, res["n_codings_done"])
# Set one coding to done
cd = cj.codings.all()[0]
cd.status = CodingStatus.objects.get(id=coding.STATUS_COMPLETE)
cd.save()
res = self.get(CodingJobResource, id=cj.id)['results'][0]
self.assertEquals(1, res["n_codings_done"])
cd.status = CodingStatus.objects.get(id=coding.STATUS_IRRELEVANT)
cd.save()
res = self.get(CodingJobResource, id=cj.id)['results'][0]
self.assertEquals(1, res["n_codings_done"])
def test_include_uncoded_articles(self):
aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(
isarticleschema=True)
sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(
isarticleschema=False)
cjob = amcattest.create_test_job(2,
articleschema=aschema,
unitschema=sschema)
a1, a2 = cjob.articleset.articles.all()
coding = create_test_coding(codingjob=cjob, article=a1)
coding.update_values({sstrf: "bla", sintf: 10})
# Default settings should not export uncoded article (a2)
fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
result = self._get_results([cjob],
fields,
export_level=CODING_LEVEL_BOTH)
self.assertEqual(1, len(result))
# Should export extra article if asked to
fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
result = self._get_results([cjob],
fields,
include_uncoded_articles=True,
export_level=CODING_LEVEL_BOTH)
self.assertEqual(2, len(result))
def test_nqueries(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
log.info(codes)
amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
c = list(codingjobs[0].codings)[0]
script = self._get_results_script([job], {strf: {}, intf: {}})
with self.checkMaxQueries(9):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf: {}, intf: {}, codef: dict(ids=True)})
with self.checkMaxQueries(9):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf: {}, intf: {}, codef: dict(labels=True)})
with self.checkMaxQueries(9):
list(csv.reader(StringIO(script.run())))
def test_get_n_done_jobs(self):
from amcat.tools import amcattest
from amcat.models.coding.codedarticle import CodedArticleStatus, STATUS_INPROGRESS
codingjob = amcattest.create_test_job(10)
s = self._get_serializer(codingjob)
self.assertEqual(0, s.get_n_done_jobs(codingjob))
self.assertEqual(10, codingjob.coded_articles.all().count())
ca, ca2, ca3 = codingjob.coded_articles.all()[0:3]
ca.status = CodedArticleStatus.objects.get(id=STATUS_COMPLETE)
ca.save()
s = self._get_serializer(codingjob)
self.assertEqual(1, s.get_n_done_jobs(codingjob))
self.assertEqual(10, codingjob.coded_articles.all().count())
ca2.status = CodedArticleStatus.objects.get(id=STATUS_IRRELEVANT)
ca2.save()
s = self._get_serializer(codingjob)
self.assertEqual(2, s.get_n_done_jobs(codingjob))
self.assertEqual(10, codingjob.coded_articles.all().count())
ca2.status = CodedArticleStatus.objects.get(id=STATUS_INPROGRESS)
ca2.save()
self.assertEqual(2, s.get_n_done_jobs(codingjob))
def test_results(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook,
isarticleschema=True)
sschema, codebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=sschema, articleschema=schema, narticles=5)
articles = list(job.articleset.articles.all())
c = amcattest.create_test_coding(codingjob=job, article=articles[0])
# test simple coding with a codebook code
c.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
self.assertEqual(self._get_results([job], {strf: {}, intf: {}, codef: dict(ids=True)}),
[('bla', 1, codes["A1b"].id)])
# test multiple codings and parents
c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
c2.update_values({strf: "blx", intf: 1, codef: codes["B1"].id})
self.assertEqual(set(self._get_results([job], {strf: {}, intf: {}, codef: dict(labels=True, parents=2)})),
{('bla', 1, "A", "A1", "A1b"), ('blx', 1, "B", "B1", "B1")})
# test sentence result
s = amcattest.create_test_sentence(article=articles[0])
sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
sc.update_values({sstrf: "z", sintf: -1, scodef: codes["A"].id})
self.assertEqual(set(self._get_results([job], {strf: {}, sstrf: {}, sintf: {}}, export_level=2)),
{('bla', 'z', -1), ('blx', None, None)})
def test_get_rows(self):
schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields()
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5)
articles = list(job.articleset.articles.all())
c = amcattest.create_test_coding(codingjob=job, article=articles[0])
ca = job.get_coded_article(articles[0])
# simple coding
rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], None, c, None)})
# test uncoded_articles
rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=True))
self.assertEqual(rows,
{(job, ca, articles[0], None, c, None)} | {(job, job.get_coded_article(a), a, None, None, None)
for a in articles[1:]})
# test sentence
s = amcattest.create_test_sentence(article=articles[0])
sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], None, c, None)})
rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], s, c, sc)})
# multiple sentence codings on the same article should duplicate article(coding)
s2 = amcattest.create_test_sentence(article=articles[0])
sc2 = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s2)
rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2)})
# if an article contains an article coding but no sentence coding, it should still show up with sentence=True
c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2),
(job, job.get_coded_article(articles[1]), articles[1], None, c2, None)})
def test_nqueries(self):
from amcat.tools import amcatlogging
amcatlogging.setup()
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
log.info(codes)
amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
c = list(codingjobs[0].codings)[0]
amcatlogging.debug_module('django.db.backends')
script = self._get_results_script([job], {strf : {}, intf : {}})
with self.checkMaxQueries(8):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(ids=True)})
with self.checkMaxQueries(8):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(labels=True)})
with self.checkMaxQueries(8):
list(csv.reader(StringIO(script.run())))
def test_results(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook, isarticleschema=True)
sschema, codebook, sstrf, sintf, scodef = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=sschema, articleschema=schema, narticles=5)
articles = list(job.articleset.articles.all())
c = amcattest.create_test_coding(codingjob=job, article=articles[0])
# test simple coding with a codebook code
c.update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
self.assertEqual(self._get_results([job], {strf : {}, intf : {}, codef : dict(ids=True)}),
[('bla', 1, codes["A1b"].id)])
# test multiple codings and parents
c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
c2.update_values({strf:"blx", intf:1, codef:codes["B1"].id})
self.assertEqual(set(self._get_results([job], {strf : {}, intf : {}, codef : dict(labels=True, parents=2)})),
{('bla', 1, "A", "A1", "A1b"), ('blx', 1, "B", "B1", "B1")})
# test sentence result
s = amcattest.create_test_sentence(article=articles[0])
sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
sc.update_values({sstrf:"z", sintf:-1, scodef:codes["A"].id})
self.assertEqual(set(self._get_results([job], {strf : {}, sstrf : {}, sintf : {}}, export_level=2)),
{('bla', 'z', -1), ('blx', None, None)})
def test_get_rows(self):
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields()
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5)
articles = list(job.articleset.articles.all())
c = amcattest.create_test_coding(codingjob=job, article=articles[0])
ca = job.get_coded_article(articles[0])
# simple coding
rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], None, c, None)})
# test uncoded_articles
rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=True))
self.assertEqual(rows, {(job, ca, articles[0], None, c, None)} | {(job, job.get_coded_article(a), a, None, None, None)
for a in articles[1:]})
# test sentence
s = amcattest.create_test_sentence(article=articles[0])
sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], None, c, None)})
rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], s, c, sc)})
# multiple sentence codings on the same article should duplicate article(coding)
s2 = amcattest.create_test_sentence(article=articles[0])
sc2 = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s2)
rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2)})
# if an article contains an article coding but no sentence coding, it should still show up with sentence=True
c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2),
(job, job.get_coded_article(articles[1]), articles[1], None, c2, None)})
def test_get_n_done_jobs(self):
from amcat.tools import amcattest
from amcat.models.coding.codedarticle import CodedArticleStatus, STATUS_INPROGRESS
codingjob = amcattest.create_test_job(10)
s = self._get_serializer(codingjob)
self.assertEqual(0, s.get_n_done_jobs(codingjob))
self.assertEqual(10, codingjob.coded_articles.all().count())
ca, ca2, ca3 = codingjob.coded_articles.all()[0:3]
ca.status = CodedArticleStatus.objects.get(id=STATUS_COMPLETE)
ca.save()
s = self._get_serializer(codingjob)
self.assertEqual(1, s.get_n_done_jobs(codingjob))
self.assertEqual(10, codingjob.coded_articles.all().count())
ca2.status = CodedArticleStatus.objects.get(id=STATUS_IRRELEVANT)
ca2.save()
s = self._get_serializer(codingjob)
self.assertEqual(2, s.get_n_done_jobs(codingjob))
self.assertEqual(10, codingjob.coded_articles.all().count())
ca2.status = CodedArticleStatus.objects.get(id=STATUS_INPROGRESS)
ca2.save()
self.assertEqual(2, s.get_n_done_jobs(codingjob))
def test_nqueries_sentence_codings(self):
aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(
isarticleschema=True)
sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(
isarticleschema=False)
cjob = amcattest.create_test_job(10,
articleschema=aschema,
unitschema=sschema)
for article in cjob.articleset.articles.all():
coding = create_test_coding(codingjob=cjob, article=article)
coding.update_values({astrf: "blas", aintf: 20})
for sentence in get_or_create_sentences(article):
coding = create_test_coding(codingjob=cjob,
article=article,
sentence=sentence)
coding.update_values({sstrf: "bla", sintf: 10})
fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
script = self._get_results_script([cjob],
fields,
export_level=CODING_LEVEL_BOTH)
with self.checkMaxQueries(9):
list(csv.reader(StringIO(script.run())))
def test_include_uncoded_sentences(self):
aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(
isarticleschema=True)
sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(
isarticleschema=False)
a1 = amcattest.create_test_article(text="Zin 1. Zin 2.")
a2 = amcattest.create_test_article(text="Zin 1. Zin 2.")
aset = amcattest.create_test_set([a1, a2])
cjob = amcattest.create_test_job(articleset=aset,
articleschema=aschema,
unitschema=sschema)
sentence = list(get_or_create_sentences(a1))[1]
coding = create_test_coding(codingjob=cjob,
article=a1,
sentence=sentence)
coding.update_values({sstrf: "bla", sintf: 10})
# We expect 1 sentence if we only export codings
fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
result = self._get_results([cjob],
fields,
include_uncoded_sentences=False,
export_level=CODING_LEVEL_BOTH)
self.assertEqual(1, len(result))
result = self._get_results([cjob],
fields,
include_uncoded_sentences=True,
export_level=CODING_LEVEL_BOTH)
self.assertEqual(3, len(result))
def test_unicode_excel(self):
"""Test whether the export can handle unicode in column names and cell values"""
try:
import openpyxl
except ImportError:
raise unittest.SkipTest(
"OpenPyxl not installed, skipping excel test")
schema = amcattest.create_test_schema(isarticleschema=True)
s1 = u'S1 \xc4\u0193 \u02a2 \u038e\u040e'
s2 = u'S2 \u053e\u06a8 \u090c \u0b8f\u0c8a'
f = CodingSchemaField.objects.create(codingschema=schema,
fieldnr=1,
label=s1,
fieldtype_id=1,
codebook=None)
job = amcattest.create_test_job(unitschema=schema,
articleschema=schema,
narticles=5)
articles = list(job.articleset.articles.all())
coding = amcattest.create_test_coding(codingjob=job,
article=articles[0])
coding.update_values({f: s2})
# test excel, can't test content but we can test output and no error
s = self._get_results_script([job], {f: {}}, export_format='xlsx')
self.assertTrue(s.run())
def test_nqueries(self):
from amcat.tools import amcatlogging
amcatlogging.setup()
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
c = codingjobs[0].codings.all()[0]
amcatlogging.debug_module('django.db.backends')
script = self._get_results_script([job], {strf : {}, intf : {}})
with self.checkMaxQueries(5):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(ids=True)})
with self.checkMaxQueries(5):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(labels=True)})
with self.checkMaxQueries(5):
list(csv.reader(StringIO(script.run())))
def test_unicode(self):
"""Test whether the export can handle unicode in column names and cell values"""
schema = amcattest.create_test_schema(isarticleschema=True)
s1 = 'S1 \xc4\u0193 \u02a2 \u038e\u040e'
s2 = 'S2 \u053e\u06a8 \u090c \u0b8f\u0c8a'
f = CodingSchemaField.objects.create(codingschema=schema,
fieldnr=1,
label=s1,
fieldtype_id=1,
codebook=None)
job = amcattest.create_test_job(unitschema=schema,
articleschema=schema,
narticles=5)
articles = list(job.articleset.articles.all())
amcattest.create_test_coding(
codingjob=job, article=articles[0]).update_values({f: s2})
# test csv
s = self._get_results_script([job], {f: {}}, export_format='csv')
data = base64.b64decode(s.run()['data']).decode('utf-8')
table = [[cell for cell in row] for row in csv.reader(StringIO(data))]
self.assertEqual(table, [[s1], [s2]])
# test json
s = self._get_results_script([job], {f: {}}, export_format='json')
self.assertEqual(json.loads(s.run()),
[[s2]]) # json export has no header (?)
def setUp(self):
from amcat.models.coding.coding import CodingValue
# create a coding job set with a sensible schema and some articles to 'code'
self.schema = amcattest.create_test_schema()
self.codebook = amcattest.create_test_codebook()
self.code = amcattest.create_test_code(label="CODED")
self.codebook.add_code(self.code)
texttype = CodingSchemaFieldType.objects.get(pk=1)
inttype = CodingSchemaFieldType.objects.get(pk=2)
codetype = CodingSchemaFieldType.objects.get(pk=5)
create = CodingSchemaField.objects.create
self.textfield = create(codingschema=self.schema,
fieldnr=1,
fieldtype=texttype,
label="Text")
self.intfield = create(codingschema=self.schema,
fieldnr=2,
fieldtype=inttype,
label="Number")
self.codefield = create(codingschema=self.schema,
fieldnr=3,
fieldtype=codetype,
label="Code",
codebook=self.codebook)
self.users = [amcattest.create_test_user() for _x in range(2)]
self.articles, self.jobs, self.asets = [], [], []
for i, user in enumerate([0, 0, 0, 0, 1]):
aset = amcattest.create_test_set(articles=2 * (i + 1))
self.articles += list(aset.articles.all())
self.asets.append(aset)
job = amcattest.create_test_job(articleschema=self.schema,
unitschema=self.schema,
coder=self.users[user],
articleset=aset)
self.jobs.append(job)
self.an1 = Coding.objects.create(codingjob=self.jobs[0],
article=self.articles[0])
self.an2 = Coding.objects.create(codingjob=self.jobs[0],
article=self.articles[1])
self.an2.set_status(STATUS_COMPLETE)
self.an2.comments = 'Makkie!'
self.an2.save()
sent = amcattest.create_test_sentence()
self.sa1 = Coding.objects.create(codingjob=self.jobs[0],
article=self.articles[0],
sentence=sent)
self.sa2 = Coding.objects.create(codingjob=self.jobs[0],
article=self.articles[0],
sentence=sent)
create = CodingValue.objects.create
create(coding=self.sa1, field=self.intfield, intval=1)
create(coding=self.sa1, field=self.textfield, strval="bla")
create(coding=self.sa2, field=self.textfield, strval="blx")
create(coding=self.sa1, field=self.codefield, intval=self.code.id)
def test_api(self):
from amcat.models import CodingStatus
cj = amcattest.create_test_job()
# Test empty codingjob
res = self.get(CodingJobResource, id=cj.id)["results"][0]
self.assertTrue("n_codings_done" in res)
self.assertTrue("n_articles" in res)
self.assertEquals(1, res["n_articles"])
self.assertEquals(0, res["n_codings_done"])
# Add two codings
cj.codings.add(amcattest.create_test_coding(), amcattest.create_test_coding())
res = self.get(CodingJobResource, id=cj.id)["results"][0]
self.assertEquals(1, res["n_articles"])
self.assertEquals(0, res["n_codings_done"])
# Set one coding to done
cd = cj.codings.all()[0]
cd.status = CodingStatus.objects.get(id=coding.STATUS_COMPLETE)
cd.save()
res = self.get(CodingJobResource, id=cj.id)["results"][0]
self.assertEquals(1, res["n_codings_done"])
cd.status = CodingStatus.objects.get(id=coding.STATUS_IRRELEVANT)
cd.save()
res = self.get(CodingJobResource, id=cj.id)["results"][0]
self.assertEquals(1, res["n_codings_done"])
def _test_caching(self):
"""DISABLED: Queries not registered??"""
from django.core.urlresolvers import reverse
cj = amcattest.create_test_job()
req = self.factory.get(reverse("api-v4-codingjob"))
with self.checkMaxQueries(1):
res = CodingJobResource().dispatch(req)
def test_n_queries(self):
from amcat.tools import amcattest
codingjob1 = amcattest.create_test_job(10)
codingjob2 = amcattest.create_test_job(5)
jobs = CodingJob.objects.filter(id__in=[codingjob1.id, codingjob2.id])
s = self._get_serializer(jobs)
# Number of codingjobs should be cached for all codingsjobs after one call
with self.checkMaxQueries(1):
s.get_n_articles(codingjob1)
s.get_n_articles(codingjob2)
# Same for done jobs
s = self._get_serializer(jobs)
with self.checkMaxQueries(1):
s.get_n_done_jobs(codingjob1)
s.get_n_done_jobs(codingjob2)
def _test_caching(self):
"""DISABLED: Queries not registered??"""
from django.core.urlresolvers import reverse
cj = amcattest.create_test_job()
req = self.factory.get(reverse("api-v4-codingjob"))
with self.checkMaxQueries(1):
res = CodingJobResource().dispatch(req)
def test_n_queries(self):
from amcat.tools import amcattest
codingjob1 = amcattest.create_test_job(10)
codingjob2 = amcattest.create_test_job(5)
jobs = CodingJob.objects.filter(id__in=[codingjob1.id, codingjob2.id])
s = self._get_serializer(jobs)
# Number of codingjobs should be cached for all codingsjobs after one call
with self.checkMaxQueries(1):
s.get_n_articles(codingjob1)
s.get_n_articles(codingjob2)
# Same for done jobs
s = self._get_serializer(jobs)
with self.checkMaxQueries(1):
s.get_n_done_jobs(codingjob1)
s.get_n_done_jobs(codingjob2)
def test_fields(self):
c = amcattest.create_test_job()
a = c.articleset.articles.all()[0]
ca = c.coded_articles.all()[0]
s = self._get_serializer(c.coded_articles.all())
self.assertEqual(a.headline, s.get_headline(ca))
self.assertEqual(a.date, s.get_date(ca))
self.assertEqual(a.pagenr, s.get_pagenr(ca))
self.assertEqual(a.length, s.get_length(ca))
def test_n_queries(self):
c = amcattest.create_test_job(10)
s = self._get_serializer(c.coded_articles.all())
ca1, ca2, ca3 = c.coded_articles.all()[0:3]
with self.checkMaxQueries(1):
s.get_title(ca1)
s.get_title(ca2)
s.get_title(ca3)
s.get_date(ca3)
def test_fields(self):
c = amcattest.create_test_job()
a = c.articleset.articles.all()[0]
ca = c.coded_articles.all()[0]
s = self._get_serializer(c.coded_articles.all())
self.assertEqual(a.headline, s.get_headline(ca))
self.assertEqual(a.date, s.get_date(ca))
self.assertEqual(a.pagenr, s.get_pagenr(ca))
self.assertEqual(a.length, s.get_length(ca))
def test_create(self):
"""Can we create a coding job with articles?"""
p = amcattest.create_test_project()
j = amcattest.create_test_job(project=p)
self.assertIsNotNone(j)
self.assertEqual(j.project, Project.objects.get(pk=p.id))
j.articleset.add(amcattest.create_test_article())
j.articleset.add(amcattest.create_test_article())
j.articleset.add(amcattest.create_test_article())
self.assertEqual(1 + 3, len(j.articleset.articles.all()))
def test_create(self):
"""Can we create an coding?"""
schema2 = amcattest.create_test_schema()
j = amcattest.create_test_job(unitschema=self.schema, articleschema=schema2)
a = amcattest.create_test_coding(codingjob=j)
self.assertIsNotNone(a)
self.assertIn(a.coded_article.article, j.articleset.articles.all())
self.assertEqual(a.schema, schema2)
a2 = amcattest.create_test_coding(codingjob=j,
sentence=amcattest.create_test_sentence())
self.assertEqual(a2.schema, self.schema)
def test_add_codedarticles(self):
"""Does add() also update codingjobs?"""
cj = amcattest.create_test_job(3)
a1 = amcattest.create_test_article()
self.assertEqual(3, cj.articleset.articles.all().count())
self.assertEqual(3, CodedArticle.objects.filter(codingjob=cj).count())
cj.articleset.add_articles([a1])
self.assertEqual(4, cj.articleset.articles.all().count())
self.assertEqual(4, CodedArticle.objects.filter(codingjob=cj).count())
def test_add_codedarticles(self):
"""Does add() also update codingjobs?"""
cj = amcattest.create_test_job(3)
a1 = amcattest.create_test_article()
self.assertEqual(3, cj.articleset.articles.all().count())
self.assertEqual(3, CodedArticle.objects.filter(codingjob=cj).count())
cj.articleset.add_articles([a1])
self.assertEqual(4, cj.articleset.articles.all().count())
self.assertEqual(4, CodedArticle.objects.filter(codingjob=cj).count())
def test_create(self):
"""Can we create a coding job with articles?"""
from amcat.models.project import Project
p = amcattest.create_test_project()
j = amcattest.create_test_job(project=p)
self.assertIsNotNone(j)
self.assertEqual(j.project, Project.objects.get(pk=p.id))
j.articleset.add(amcattest.create_test_article())
j.articleset.add(amcattest.create_test_article())
j.articleset.add(amcattest.create_test_article())
self.assertEqual(1+3, len(j.articleset.articles.all()))
def test_n_queries(self):
c = amcattest.create_test_job(10)
s = self._get_serializer(c.coded_articles.all())
ca1, ca2, ca3 = c.coded_articles.all()[0:3]
with self.checkMaxQueries(1):
s.get_title(ca1)
s.get_title(ca2)
s.get_title(ca3)
s.get_date(ca3)
s.get_pagenr(ca3)
def test_create(self):
"""Can we create an coding?"""
schema2 = amcattest.create_test_schema()
j = amcattest.create_test_job(unitschema=self.schema, articleschema=schema2)
a = amcattest.create_test_coding(codingjob=j)
self.assertIsNotNone(a)
self.assertIn(a.article, j.articleset.articles.all())
self.assertEqual(a.schema, schema2)
a2 = amcattest.create_test_coding(codingjob=j,
sentence=amcattest.create_test_sentence())
self.assertEqual(a2.schema, self.schema)
def test_replace_codings(self):
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(isarticleschema=True)
schema2, codebook2, strf2, intf2, codef2 = amcattest.create_test_schema_with_fields(isarticleschema=True)
codingjob = amcattest.create_test_job(articleschema=schema, narticles=10)
coded_article = CodedArticle.objects.get(article=codingjob.articleset.articles.all()[0], codingjob=codingjob)
coded_article.replace_codings([self._get_coding_dict(intval=10, field_id=codef.id)])
self.assertEqual(1, coded_article.codings.all().count())
self.assertEqual(1, coded_article.codings.all()[0].values.all().count())
coding = coded_article.codings.all()[0]
value = coding.values.all()[0]
self.assertEqual(coding.sentence, None)
self.assertEqual(value.strval, None)
self.assertEqual(value.intval, 10)
self.assertEqual(value.field, codef)
# Overwrite previous coding
coded_article.replace_codings([self._get_coding_dict(intval=11, field_id=intf.id)])
self.assertEqual(1, coded_article.codings.all().count())
self.assertEqual(1, coded_article.codings.all()[0].values.all().count())
coding = coded_article.codings.all()[0]
value = coding.values.all()[0]
self.assertEqual(coding.sentence, None)
self.assertEqual(value.strval, None)
self.assertEqual(value.intval, 11)
self.assertEqual(value.field, intf)
# Try to insert illigal values
illval1 = self._get_coding_dict(intval=1, strval="a", field_id=intf.id)
illval2 = self._get_coding_dict(field_id=intf.id)
illval3 = self._get_coding_dict(intval=1)
illval4 = self._get_coding_dict(intval=1, field_id=strf2.id)
self.assertRaises(ValueError, coded_article.replace_codings, [illval1])
self.assertRaises(ValueError, coded_article.replace_codings, [illval2])
self.assertRaises(IntegrityError, coded_article.replace_codings, [illval3])
self.assertRaises(ValueError, coded_article.replace_codings, [illval4])
# Unspecified values default to None
val = self._get_coding_dict(intval=1, field_id=intf.id)
del val["values"][0]["strval"]
coded_article.replace_codings([val])
value = coded_article.codings.all()[0].values.all()[0]
self.assertEqual(value.strval, None)
self.assertEqual(value.intval, 1)
val = self._get_coding_dict(strval="a", field_id=intf.id)
del val["values"][0]["intval"]
coded_article.replace_codings([val])
value = coded_article.codings.all()[0].values.all()[0]
self.assertEqual(value.strval, "a")
self.assertEqual(value.intval, None)
def _create_job(self, fieldtypes):
"""Create a job whose schemas contain the given fields"""
schema = amcattest.create_test_schema()
for i, t in enumerate(fieldtypes):
CodingSchemaField.objects.create(codingschema=schema,
fieldnr=i,
fieldtype=t,
label="field%i" % i,
codebook=self.codebook)
return amcattest.create_test_job(articleschema=schema,
unitschema=schema)
def test_fields(self):
c = amcattest.create_test_job()
a = c.articleset.articles.all()[0]
a.set_property("length_int", 3)
a.set_property("pagenr_int", 3)
a.save()
ca = c.coded_articles.all()[0]
s = self._get_serializer(c.coded_articles.all())
self.assertEqual(a.title, s.get_title(ca))
self.assertEqual(a.date, s.get_date(ca))
def setUp(self):
"""Set up a simple codingschema with fields to use for testing"""
super(TestCoding, self).setUp()
self.schema, self.codebook, self.strfield, self.intfield, self.codefield, _, _ = (
amcattest.create_test_schema_with_fields())
self.c = amcattest.create_test_code(label="CODED")
self.c2 = amcattest.create_test_code(label="CODE2")
self.codebook.add_code(self.c)
self.codebook.add_code(self.c2)
self.job = amcattest.create_test_job(unitschema=self.schema, articleschema=self.schema)
def setUp(self):
"""Set up a simple coding schema with fields to use for testing"""
super(TestCoding, self).setUp()
self.schema, self.codebook, self.strfield, self.intfield, self.codefield = (
amcattest.create_test_schema_with_fields())
self.c = amcattest.create_test_code(label="CODED")
self.c2 = amcattest.create_test_code(label="CODE2")
self.codebook.add_code(self.c)
self.codebook.add_code(self.c2)
self.job = amcattest.create_test_job(unitschema=self.schema, articleschema=self.schema)
def test_delete(self):
"""Simple deletion of a job"""
from amcat.models import ArticleSet, Coding
s = amcattest.create_test_set(articles=5)
j = amcattest.create_test_job(articleset=s)
c = amcattest.create_test_coding(codingjob=j)
self.assertTrue(CodingJob.objects.filter(pk=j.id).exists())
self.assertTrue(ArticleSet.objects.filter(pk=s.id).exists())
self.assertTrue(Coding.objects.filter(pk=c.id).exists())
DeleteCodingJob(job=j.id).run()
self.assertFalse(CodingJob.objects.filter(pk=j.id).exists())
self.assertFalse(ArticleSet.objects.filter(pk=s.id).exists())
self.assertFalse(Coding.objects.filter(pk=c.id).exists())
def test_delete(self):
"""Simple deletion of a job"""
from amcat.models import ArticleSet, Coding
s = amcattest.create_test_set(articles=5)
j = amcattest.create_test_job(articleset=s)
c = amcattest.create_test_coding(codingjob=j)
self.assertTrue(CodingJob.objects.filter(pk=j.id).exists())
self.assertTrue(ArticleSet.objects.filter(pk=s.id).exists())
self.assertTrue(Coding.objects.filter(pk=c.id).exists())
DeleteCodingJob(job=j.id).run()
self.assertFalse(CodingJob.objects.filter(pk=j.id).exists())
self.assertFalse(ArticleSet.objects.filter(pk=s.id).exists())
self.assertFalse(Coding.objects.filter(pk=c.id).exists())
def test_fields(self):
c = amcattest.create_test_job()
a = c.articleset.articles.all()[0]
a.set_property("length_int", 3)
a.set_property("pagenr_int", 3)
a.save()
ca = c.coded_articles.all()[0]
s = self._get_serializer(c.coded_articles.all())
self.assertEqual(a.title, s.get_title(ca))
self.assertEqual(a.date, s.get_date(ca))
self.assertEqual(a.get_property("pagenr_int"), s.get_pagenr(ca))
def test_nqueries(self):
"""Does getting a table of values not use too many queries?"""
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields()
job = amcattest.create_test_job(unitschema=schema, articleschema=schema)
for i in range(10):
c = amcattest.create_test_coding(codingjob=job)
c.update_values({strf:"bla %i"%i, intf:i})
job = CodingJob.objects.get(pk=job.id)
with self.checkMaxQueries(6):
# 1. get schema, 2. get codings, 3. get values, 4. get field, 5+6. get serialiser
t = job.values_table()
cells = list(t.to_list())
def setUp(self):
from amcat.models.coding.coding import CodingValue
# create a coding job set with a sensible schema and some articles to 'code'
self.schema = amcattest.create_test_schema()
self.codebook = amcattest.create_test_codebook()
self.code = amcattest.create_test_code(label="CODED")
self.codebook.add_code(self.code)
texttype = CodingSchemaFieldType.objects.get(pk=1)
inttype = CodingSchemaFieldType.objects.get(pk=2)
codetype = CodingSchemaFieldType.objects.get(pk=5)
create = CodingSchemaField.objects.create
self.textfield = create(codingschema=self.schema, fieldnr=1, fieldtype=texttype,
label="Text")
self.intfield = create(codingschema=self.schema, fieldnr=2, fieldtype=inttype,
label="Number")
self.codefield = create(codingschema=self.schema, fieldnr=3, fieldtype=codetype,
label="Code", codebook=self.codebook)
self.users = [amcattest.create_test_user() for _x in range(2)]
self.articles, self.jobs, self.asets = [], [], []
for i, user in enumerate([0, 0, 0, 0, 1]):
aset = amcattest.create_test_set(articles=2 * (i+1))
self.articles += list(aset.articles.all())
self.asets.append(aset)
job = amcattest.create_test_job(articleschema=self.schema, unitschema=self.schema,
coder = self.users[user], articleset=aset)
self.jobs.append(job)
self.an1 = Coding.objects.create(codingjob=self.jobs[0], article=self.articles[0])
self.an2 = Coding.objects.create(codingjob=self.jobs[0], article=self.articles[1])
self.an2.set_status(STATUS_COMPLETE)
self.an2.comments = 'Makkie!'
self.an2.save()
sent = amcattest.create_test_sentence()
self.sa1 = Coding.objects.create(codingjob=self.jobs[0], article=self.articles[0],
sentence=sent)
self.sa2 = Coding.objects.create(codingjob=self.jobs[0], article=self.articles[0],
sentence=sent)
create = CodingValue.objects.create
create(coding=self.sa1, field=self.intfield, intval=1)
create(coding=self.sa1, field=self.textfield, strval="bla")
create(coding=self.sa2, field=self.textfield, strval="blx")
create(coding=self.sa1, field=self.codefield, intval=self.code.id)
def setUp(self):
"""Populate a project with articles, coding etc"""
from amcat.models import Language, Article
self.project = amcattest.create_test_project()
# create a codebook
self.codebook = amcattest.create_test_codebook(project=self.project, name="base codebook")
en = Language.objects.get(label='en')
sv = Language.objects.create(label='sv')
_cargs = dict(language=en, codebook=self.codebook)
self.code_a = amcattest.create_test_code(label="a", **_cargs)
self.code_a.add_label(label=u"\xe5", language=sv)
self.code_b = amcattest.create_test_code(label="b", parent=self.code_a, **_cargs)
self.code_c = amcattest.create_test_code(label="c", parent=self.code_a, **_cargs)
# Codebook bases not yet implemented
self.sub_codebook = amcattest.create_test_codebook(project=self.project, name="sub codebook")
#self.sub_codebook.add_base(self.codebook)
self.code_d = amcattest.create_test_code(label="d", language=en,
codebook=self.sub_codebook, parent=self.code_a)
CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_c, hide=True)
CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_b, parent=None)
# create a schema
self.schema, _dummy, self.strfield, self.intfield, self.codefield = (
amcattest.create_test_schema_with_fields(project=self.project, codebook=self.sub_codebook))
self.article_hl = u'The great wall of China (\u9577\u57ce)'
self.article_text = u"""This is some text with greek characters\n
\u03bc\u1fc6\u03bd\u03b9\u03bd \u1f04\u03b5\u03b9\u03b4\u03b5,
\u03b8\u03b5\u03ac,
\u03a0\u03b7\u03bb\u03b7\u03ca\u03ac\u03b4\u03b5\u03c9
\u1f08\u03c7\u03b9\u03bb\u1fc6\u03bf\u03c2"""
self.article = amcattest.create_test_article(headline=self.article_hl, project=self.project,
text=self.article_text, date="2012-01-01")
self.article = Article.objects.get(pk=self.article.id) # to get date redeserialized
self.articleset = amcattest.create_test_set(project=self.project)
self.articleset.add(self.article)
self.job = amcattest.create_test_job(project=self.project, articleset=self.articleset,
unitschema=self.schema, articleschema=self.schema)
self.coding = amcattest.create_test_coding(codingjob=self.job, comments="Obvious", article=self.article)
self.coding.update_values({self.strfield:"abc", self.intfield:1, self.codefield:self.code_d})
def test_nqueries_sentence_codings(self):
aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True)
sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False)
cjob = amcattest.create_test_job(10, articleschema=aschema, unitschema=sschema)
for article in cjob.articleset.articles.all():
coding = create_test_coding(codingjob=cjob, article=article)
coding.update_values({astrf: "blas", aintf: 20})
for sentence in get_or_create_sentences(article):
coding = create_test_coding(codingjob=cjob, article=article, sentence=sentence)
coding.update_values({sstrf: "bla", sintf: 10})
fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
script = self._get_results_script([cjob], fields, export_level=CODING_LEVEL_BOTH)
with self.checkMaxQueries(9):
list(csv.reader(StringIO(script.run())))
def test_include_uncoded_articles(self):
aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True)
sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False)
cjob = amcattest.create_test_job(2, articleschema=aschema, unitschema=sschema)
a1, a2 = cjob.articleset.articles.all()
coding = create_test_coding(codingjob=cjob, article=a1)
coding.update_values({sstrf: "bla", sintf: 10})
# Default settings should not export uncoded article (a2)
fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
result = self._get_results([cjob], fields, export_level=CODING_LEVEL_BOTH)
self.assertEqual(1, len(result))
# Should export extra article if asked to
fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
result = self._get_results([cjob], fields, include_uncoded_articles=True, export_level=CODING_LEVEL_BOTH)
self.assertEqual(2, len(result))
def test_values_table(self):
"""Does getting a table of values work?"""
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields()
job = amcattest.create_test_job(unitschema=schema, articleschema=schema)
c = amcattest.create_test_coding(codingjob=job)
c.update_values({strf:"bla", intf:1})
self.assertEqual(set(job.values_table().to_list()), {('bla', 1, None)})
code = amcattest.create_test_code(label="CODED")
codebook.add_code(code)
c2 = amcattest.create_test_coding(codingjob=job)
c2.update_values({intf:-1, codef: code})
t = job.values_table()
self.assertEqual(set(t.rows), {c, c2})
self.assertEqual(set(t.to_list()), {('bla', 1, None), (None, -1, code.id)})
def test_include_uncoded_sentences(self):
aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True)
sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False)
a1 = amcattest.create_test_article(text="Zin 1. Zin 2.")
a2 = amcattest.create_test_article(text="Zin 1. Zin 2.")
aset = amcattest.create_test_set([a1, a2])
cjob = amcattest.create_test_job(articleset=aset, articleschema=aschema, unitschema=sschema)
sentence = list(get_or_create_sentences(a1))[1]
coding = create_test_coding(codingjob=cjob, article=a1, sentence=sentence)
coding.update_values({sstrf: "bla", sintf: 10})
# We expect 1 sentence if we only export codings
fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
result = self._get_results([cjob], fields, include_uncoded_sentences=False, export_level=CODING_LEVEL_BOTH)
self.assertEqual(1, len(result))
result = self._get_results([cjob], fields, include_uncoded_sentences=True, export_level=CODING_LEVEL_BOTH)
self.assertEqual(3, len(result))
def test_update_values(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
coding = amcattest.create_test_coding(codingjob=job, article=articles[0])
self.assertEqual(0, coding.values.count())
coding.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
self.assertEqual(3, coding.values.count())
self.assertTrue(strf in dict(coding.get_values()))
self.assertTrue(intf in dict(coding.get_values()))
self.assertTrue(codef in dict(coding.get_values()))
self.assertEqual(1, dict(coding.get_values())[intf])
# Does update_values delete values not present in dict?
coding.update_values({strf: "blas"})
self.assertEqual(1, coding.values.count())
self.assertTrue(strf in dict(coding.get_values()))
self.assertEqual("blas", dict(coding.get_values())[strf])
def test_update_values(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
coding = amcattest.create_test_coding(codingjob=job, article=articles[0])
self.assertEqual(0, coding.values.count())
coding.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
self.assertEqual(3, coding.values.count())
self.assertTrue(strf in dict(coding.get_values()))
self.assertTrue(intf in dict(coding.get_values()))
self.assertTrue(codef in dict(coding.get_values()))
self.assertEqual(1, dict(coding.get_values())[intf])
# Does update_values delete values not present in dict?
coding.update_values({strf: "blas"})
self.assertEqual(1, coding.values.count())
self.assertTrue(strf in dict(coding.get_values()))
self.assertEqual("blas", dict(coding.get_values())[strf])
def test_unicode(self):
"""Test whether the export can handle unicode in column names and cell values"""
schema = amcattest.create_test_schema(isarticleschema=True)
s1 = u'S1 \xc4\u0193 \u02a2 \u038e\u040e'
s2 = u'S2 \u053e\u06a8 \u090c \u0b8f\u0c8a'
f = CodingSchemaField.objects.create(codingschema=schema, fieldnr=1, label=s1,
fieldtype_id=1, codebook=None)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5)
articles = list(job.articleset.articles.all())
amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({f:s2})
# test csv
s = self._get_results_script([job], {f : {}}, export_format='csv')
table = [[cell.decode('utf-8') for cell in row] for row in csv.reader(StringIO(s.run()))]
self.assertEqual(table, [[s1], [s2]])
# test json
s = self._get_results_script([job], {f : {}}, export_format='json')
self.assertEqual(json.loads(s.run()), [[s2]]) # json export has no header (?)
def test_unicode_excel(self):
"""Test whether the export can handle unicode in column names and cell values"""
try:
import openpyxl
except ImportError:
raise unittest.SkipTest("OpenPyxl not installed, skipping excel test")
schema = amcattest.create_test_schema(isarticleschema=True)
s1 = u'S1 \xc4\u0193 \u02a2 \u038e\u040e'
s2 = u'S2 \u053e\u06a8 \u090c \u0b8f\u0c8a'
f = CodingSchemaField.objects.create(codingschema=schema, fieldnr=1, label=s1,
fieldtype_id=1, codebook=None)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5)
articles = list(job.articleset.articles.all())
amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({f:s2})
# test excel, can't test content but we can test output and no error
s = self._get_results_script([job], {f : {}}, export_format='xlsx')
self.assertTrue(s.run())
def test_replace_codings(self):
schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(
isarticleschema=True)
schema2, codebook2, strf2, intf2, codef2, _, _ = amcattest.create_test_schema_with_fields(
isarticleschema=True)
codingjob = amcattest.create_test_job(articleschema=schema,
narticles=10)
coded_article = CodedArticle.objects.get(
article=codingjob.articleset.articles.all()[0],
codingjob=codingjob)
coded_article.replace_codings(
[self._get_coding_dict(intval=10, field_id=codef.id)])
self.assertEqual(1, coded_article.codings.all().count())
self.assertEqual(1,
coded_article.codings.all()[0].values.all().count())
coding = coded_article.codings.all()[0]
value = coding.values.all()[0]
self.assertEqual(coding.sentence, None)
self.assertEqual(value.strval, None)
self.assertEqual(value.intval, 10)
self.assertEqual(value.field, codef)
# Overwrite previous coding
coded_article.replace_codings(
[self._get_coding_dict(intval=11, field_id=intf.id)])
self.assertEqual(1, coded_article.codings.all().count())
self.assertEqual(1,
coded_article.codings.all()[0].values.all().count())
coding = coded_article.codings.all()[0]
value = coding.values.all()[0]
self.assertEqual(coding.sentence, None)
self.assertEqual(value.strval, None)
self.assertEqual(value.intval, 11)
self.assertEqual(value.field, intf)
# Try to insert illigal values
illval1 = self._get_coding_dict(intval=1, strval="a", field_id=intf.id)
illval2 = self._get_coding_dict(field_id=intf.id)
illval3 = self._get_coding_dict(intval=1)
illval4 = self._get_coding_dict(intval=1, field_id=strf2.id)
self.assertRaises(ValueError, coded_article.replace_codings, [illval1])
self.assertRaises(ValueError, coded_article.replace_codings, [illval2])
self.assertRaises(IntegrityError, coded_article.replace_codings,
[illval3])
self.assertRaises(ValueError, coded_article.replace_codings, [illval4])
# Unspecified values default to None
val = self._get_coding_dict(intval=1, field_id=intf.id)
del val["values"][0]["strval"]
coded_article.replace_codings([val])
value = coded_article.codings.all()[0].values.all()[0]
self.assertEqual(value.strval, None)
self.assertEqual(value.intval, 1)
val = self._get_coding_dict(strval="a", field_id=intf.id)
del val["values"][0]["intval"]
coded_article.replace_codings([val])
value = coded_article.codings.all()[0].values.all()[0]
self.assertEqual(value.strval, "a")
self.assertEqual(value.intval, None)
def setUp(self):
"""Populate a project with articles, coding etc"""
from amcat.models import Language, Article
self.project = amcattest.create_test_project()
# create a codebook
self.codebook = amcattest.create_test_codebook(project=self.project,
name="base codebook")
en = Language.objects.get(label='en')
sv = Language.objects.create(label='sv')
_cargs = dict(language=en, codebook=self.codebook)
self.code_a = amcattest.create_test_code(label="a", **_cargs)
self.code_a.add_label(label=u"\xe5", language=sv)
self.code_b = amcattest.create_test_code(label="b",
parent=self.code_a,
**_cargs)
self.code_c = amcattest.create_test_code(label="c",
parent=self.code_a,
**_cargs)
# Codebook bases not yet implemented
self.sub_codebook = amcattest.create_test_codebook(
project=self.project, name="sub codebook")
#self.sub_codebook.add_base(self.codebook)
self.code_d = amcattest.create_test_code(label="d",
language=en,
codebook=self.sub_codebook,
parent=self.code_a)
CodebookCode.objects.create(codebook=self.sub_codebook,
code=self.code_c,
hide=True)
CodebookCode.objects.create(codebook=self.sub_codebook,
code=self.code_b,
parent=None)
# create a schema
self.schema, _dummy, self.strfield, self.intfield, self.codefield = (
amcattest.create_test_schema_with_fields(
project=self.project, codebook=self.sub_codebook))
self.article_hl = u'The great wall of China (\u9577\u57ce)'
self.article_text = u"""This is some text with greek characters\n
\u03bc\u1fc6\u03bd\u03b9\u03bd \u1f04\u03b5\u03b9\u03b4\u03b5,
\u03b8\u03b5\u03ac,
\u03a0\u03b7\u03bb\u03b7\u03ca\u03ac\u03b4\u03b5\u03c9
\u1f08\u03c7\u03b9\u03bb\u1fc6\u03bf\u03c2"""
self.article = amcattest.create_test_article(headline=self.article_hl,
project=self.project,
text=self.article_text,
date="2012-01-01")
self.article = Article.objects.get(
pk=self.article.id) # to get date redeserialized
self.articleset = amcattest.create_test_set(project=self.project)
self.articleset.add(self.article)
self.job = amcattest.create_test_job(project=self.project,
articleset=self.articleset,
unitschema=self.schema,
articleschema=self.schema)
self.coding = amcattest.create_test_coding(codingjob=self.job,
comments="Obvious",
article=self.article)
self.coding.update_values({
self.strfield: "abc",
self.intfield: 1,
self.codefield: self.code_d
})
def test_post_create(self):
job = amcattest.create_test_job(10)
self.assertEqual(
CodedArticle.objects.filter(codingjob=job).count(), 10)
def setUp(self):
self.s1 = amcattest.create_test_set(5)
self.a1, self.a2, self.a3, self.a4, self.a5 = self.s1.articles.all()
self.m1 = self.a1.medium
self.m2 = self.a2.medium
self.m3 = self.a3.medium
self.m4 = self.a4.medium
self.a3.medium = self.m2
self.a3.save()
self.a1.text = "aap."
self.a2.text = "aap. noot."
self.a3.text = "aap. noot. mies."
self.a1.date = datetime.datetime(2015, 0o1, 0o1)
self.a2.date = datetime.datetime(2015, 0o1, 0o1)
self.a3.date = datetime.datetime(2015, 0o2, 0o1)
self.a4.date = datetime.datetime(2016, 0o1, 0o1)
self.a5.date = datetime.datetime(2016, 0o1, 0o1)
self.a1.save()
self.a2.save()
self.a3.save()
self.a4.save()
self.a5.save()
# Uncomment if ever using elastic :)
# self.s1.refresh_index(full_refresh=True)
self.schema, self.codebook, self.strf, self.intf, self.codef, self.boolf, self.qualf = (
amcattest.create_test_schema_with_fields(isarticleschema=True))
self.sschema, self.scodebook, self.sstrf, self.sintf, self.scodef, self.sboolf, self.squalf = (
amcattest.create_test_schema_with_fields(isarticleschema=False))
# Article
self.codes = self.codebook.get_codes()
self.code_A, = [c for c in self.codes if c.label == "A"]
self.code_B, = [c for c in self.codes if c.label == "B"]
self.code_A1, = [c for c in self.codes if c.label == "A1"]
# Sentence
self.scodes = self.codebook.get_codes()
self.scode_A, = [c for c in self.scodes if c.label == "A"]
self.scode_B, = [c for c in self.scodes if c.label == "B"]
self.scode_A1, = [c for c in self.scodes if c.label == "A1"]
# Does not get fired in unit test?
for article in [self.a1, self.a2, self.a3, self.a4, self.a5]:
get_or_create_sentences(article)
self.job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema, unitschema=self.sschema)
self.c1 = amcattest.create_test_coding(codingjob=self.job, article=self.a1)
self.c1.update_values({self.codef: self.code_A.id, self.intf: 4, self.qualf: 4})
self.c2 = amcattest.create_test_coding(codingjob=self.job, article=self.a2)
self.c2.update_values({self.codef: self.code_A.id, self.intf: 2, self.qualf: 1})
self.c3 = amcattest.create_test_coding(codingjob=self.job, article=self.a3)
self.c3.update_values({self.codef: self.code_B.id, self.intf: 1, self.qualf: 2})
self.c4 = amcattest.create_test_coding(codingjob=self.job, article=self.a4)
self.c4.update_values({self.codef: self.code_A1.id, self.intf: 1})
self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a1, sentence=self.a1.sentences.all()[0])
self.sentence_coding.update_values({self.scodef: self.scode_A1.id, self.sintf: 1})
# Try to confuse aggregator by inserting multiple codingjobs
job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema)
c4 = amcattest.create_test_coding(codingjob=job, article=self.a3)
c4.update_values({self.codef: self.code_B.id, self.intf: 10, self.qualf: 8})
