def test_api(self): from amcat.models import CodingStatus cj = amcattest.create_test_job() # Test empty codingjob res = self.get(CodingJobResource, id=cj.id)["results"][0] self.assertTrue("n_codings_done" in res) self.assertTrue("n_articles" in res) self.assertEquals(1, res["n_articles"]) self.assertEquals(0, res["n_codings_done"]) # Add two codings cj.codings.add(amcattest.create_test_coding(), amcattest.create_test_coding()) res = self.get(CodingJobResource, id=cj.id)["results"][0] self.assertEquals(1, res["n_articles"]) self.assertEquals(0, res["n_codings_done"]) # Set one coding to done cd = cj.codings.all()[0] cd.status = CodingStatus.objects.get(id=coding.STATUS_COMPLETE) cd.save() res = self.get(CodingJobResource, id=cj.id)["results"][0] self.assertEquals(1, res["n_codings_done"]) cd.status = CodingStatus.objects.get(id=coding.STATUS_IRRELEVANT) cd.save() res = self.get(CodingJobResource, id=cj.id)["results"][0] self.assertEquals(1, res["n_codings_done"])
def test_results(self): codebook, codes = amcattest.create_test_codebook_with_codes() schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook, isarticleschema=True) sschema, codebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook) job = amcattest.create_test_job(unitschema=sschema, articleschema=schema, narticles=5) articles = list(job.articleset.articles.all()) c = amcattest.create_test_coding(codingjob=job, article=articles[0]) # test simple coding with a codebook code c.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id}) self.assertEqual(self._get_results([job], {strf: {}, intf: {}, codef: dict(ids=True)}), [('bla', 1, codes["A1b"].id)]) # test multiple codings and parents c2 = amcattest.create_test_coding(codingjob=job, article=articles[1]) c2.update_values({strf: "blx", intf: 1, codef: codes["B1"].id}) self.assertEqual(set(self._get_results([job], {strf: {}, intf: {}, codef: dict(labels=True, parents=2)})), {('bla', 1, "A", "A1", "A1b"), ('blx', 1, "B", "B1", "B1")}) # test sentence result s = amcattest.create_test_sentence(article=articles[0]) sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s) sc.update_values({sstrf: "z", sintf: -1, scodef: codes["A"].id}) self.assertEqual(set(self._get_results([job], {strf: {}, sstrf: {}, sintf: {}}, export_level=2)), {('bla', 'z', -1), ('blx', None, None)})
def test_nqueries(self): from amcat.tools import amcatlogging amcatlogging.setup() codebook, codes = amcattest.create_test_codebook_with_codes() schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7) articles = list(job.articleset.articles.all()) amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({strf:"bla", intf:1, codef:codes["A1b"]}) amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({strf:"bla", intf:1, codef:codes["A1b"]}) amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({strf:"bla", intf:1, codef:codes["A1b"]}) amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({strf:"bla", intf:1, codef:codes["A1b"]}) amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({strf:"bla", intf:1, codef:codes["A1b"]}) codingjobs = list(CodingJob.objects.filter(pk__in=[job.id])) c = codingjobs[0].codings.all()[0] amcatlogging.debug_module('django.db.backends') script = self._get_results_script([job], {strf : {}, intf : {}}) with self.checkMaxQueries(5): list(csv.reader(StringIO(script.run()))) script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(ids=True)}) with self.checkMaxQueries(5): list(csv.reader(StringIO(script.run()))) script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(labels=True)}) with self.checkMaxQueries(5): list(csv.reader(StringIO(script.run())))
def test_nqueries(self): codebook, codes = amcattest.create_test_codebook_with_codes() schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7) articles = list(job.articleset.articles.all()) log.info(codes) amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values( {strf: "bla", intf: 1, codef: codes["A1b"].id}) amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values( {strf: "bla", intf: 1, codef: codes["A1b"].id}) amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values( {strf: "bla", intf: 1, codef: codes["A1b"].id}) amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values( {strf: "bla", intf: 1, codef: codes["A1b"].id}) amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values( {strf: "bla", intf: 1, codef: codes["A1b"].id}) codingjobs = list(CodingJob.objects.filter(pk__in=[job.id])) c = list(codingjobs[0].codings)[0] script = self._get_results_script([job], {strf: {}, intf: {}}) with self.checkMaxQueries(9): list(csv.reader(StringIO(script.run()))) script = self._get_results_script([job], {strf: {}, intf: {}, codef: dict(ids=True)}) with self.checkMaxQueries(9): list(csv.reader(StringIO(script.run()))) script = self._get_results_script([job], {strf: {}, intf: {}, codef: dict(labels=True)}) with self.checkMaxQueries(9): list(csv.reader(StringIO(script.run())))
def test_get_rows(self): schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields() job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5) articles = list(job.articleset.articles.all()) c = amcattest.create_test_coding(codingjob=job, article=articles[0]) ca = job.get_coded_article(articles[0]) # simple coding rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], None, c, None)}) # test uncoded_articles rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=True)) self.assertEqual(rows, {(job, ca, articles[0], None, c, None)} | {(job, job.get_coded_article(a), a, None, None, None) for a in articles[1:]}) # test sentence s = amcattest.create_test_sentence(article=articles[0]) sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s) rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], None, c, None)}) rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], s, c, sc)}) # multiple sentence codings on the same article should duplicate article(coding) s2 = amcattest.create_test_sentence(article=articles[0]) sc2 = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s2) rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2)}) # if an article contains an article coding but no sentence coding, it should still show up with sentence=True c2 = amcattest.create_test_coding(codingjob=job, article=articles[1]) rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2), (job, job.get_coded_article(articles[1]), articles[1], None, c2, None)})
def test_create_value(self): """Can we create an coding value?""" a = amcattest.create_test_coding(codingjob=self.job) v = CodingValue.objects.create(coding=a, field=self.strfield, intval=1, strval="abc") v2 = CodingValue.objects.create(coding=a, field=self.intfield, intval=1, strval="abc") v3 = CodingValue.objects.create(coding=a, field=self.codefield, intval=self.c.id) self.assertIn(v, a.values.all()) self.assertEqual(v.value, "abc") self.assertEqual(v2.value, 1) self.assertEqual(v3.value, self.c) self.assertEqual(list(a.get_values()), [(self.strfield, "abc"), (self.intfield, 1), (self.codefield, self.c)]) # null values for both value fields self.assertRaises(ValueError, CodingValue.objects.create, coding=amcattest.create_test_coding(codingjob=self.job), field=self.strfield) # field does not exist in (newly created) schema self.assertRaises(ValueError, CodingValue.objects.create, coding=amcattest.create_test_coding(), field=self.strfield, strval="abc")
def test_codedarticle(self): """Test whether CodedArticle coding retrieval works""" a = amcattest.create_test_coding() s = amcattest.create_test_sentence() a2 = amcattest.create_test_coding(sentence=s, codingjob=a.codingjob) a3 = amcattest.create_test_coding(sentence=s, codingjob=a.codingjob) ca = CodedArticle(a) self.assertEqual(set(ca.sentence_codings), set([a2, a3])) self.assertEqual(ca.coding, a)
def test_create(self): """Can we create an coding?""" schema2 = amcattest.create_test_schema() j = amcattest.create_test_job(unitschema=self.schema, articleschema=schema2) a = amcattest.create_test_coding(codingjob=j) self.assertIsNotNone(a) self.assertIn(a.article, j.articleset.articles.all()) self.assertEqual(a.schema, schema2) a2 = amcattest.create_test_coding(codingjob=j, sentence=amcattest.create_test_sentence()) self.assertEqual(a2.schema, self.schema)
def test_nqueries_sentence_codings(self): aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True) sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False) cjob = amcattest.create_test_job(10, articleschema=aschema, unitschema=sschema) for article in cjob.articleset.articles.all(): coding = create_test_coding(codingjob=cjob, article=article) coding.update_values({astrf: "blas", aintf: 20}) for sentence in get_or_create_sentences(article): coding = create_test_coding(codingjob=cjob, article=article, sentence=sentence) coding.update_values({sstrf: "bla", sintf: 10}) fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}} script = self._get_results_script([cjob], fields, export_level=CODING_LEVEL_BOTH) with self.checkMaxQueries(9): list(csv.reader(StringIO(script.run())))
def test_values_table(self): """Does getting a table of values work?""" schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields() job = amcattest.create_test_job(unitschema=schema, articleschema=schema) c = amcattest.create_test_coding(codingjob=job) c.update_values({strf:"bla", intf:1}) self.assertEqual(set(job.values_table().to_list()), {('bla', 1, None)}) code = amcattest.create_test_code(label="CODED") codebook.add_code(code) c2 = amcattest.create_test_coding(codingjob=job) c2.update_values({intf:-1, codef: code}) t = job.values_table() self.assertEqual(set(t.rows), {c, c2}) self.assertEqual(set(t.to_list()), {('bla', 1, None), (None, -1, code.id)})
def test_create_value(self): """Can we create an coding value?""" a = amcattest.create_test_coding(codingjob=self.job) v = CodingValue.objects.create(coding=a, field=self.strfield, strval="abc") v2 = CodingValue.objects.create(coding=a, field=self.intfield, intval=1) v3 = CodingValue.objects.create(coding=a, field=self.codefield, intval=self.c.id) self.assertIn(v, a.values.all()) self.assertEqual(v.value, "abc") self.assertEqual(v2.value, 1) self.assertEqual(v3.value, self.c) self.assertEqual(list(a.get_values()), [(self.strfield, "abc"), (self.intfield, 1), (self.codefield, self.c)]) # null values for both value fields self.assertRaises(ValueError, CodingValue.objects.create, coding=amcattest.create_test_coding(codingjob=self.job), field=self.strfield)
def test_comments(self): """Can we set and read comments?""" a = amcattest.create_test_coding() self.assertIsNone(a.comments) for offset in range(4563, 20000, 1000): s = "".join(unichr(offset + c) for c in range(12, 1000, 100)) a.comments = s a.save() a = Coding.objects.get(pk=a.id) self.assertEqual(a.comments, s)
def test_unicode(self): """Test whether the export can handle unicode in column names and cell values""" schema = amcattest.create_test_schema(isarticleschema=True) s1 = u'S1 \xc4\u0193 \u02a2 \u038e\u040e' s2 = u'S2 \u053e\u06a8 \u090c \u0b8f\u0c8a' f = CodingSchemaField.objects.create(codingschema=schema, fieldnr=1, label=s1, fieldtype_id=1, codebook=None) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5) articles = list(job.articleset.articles.all()) amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({f:s2}) # test csv s = self._get_results_script([job], {f : {}}, export_format='csv') table = [[cell.decode('utf-8') for cell in row] for row in csv.reader(StringIO(s.run()))] self.assertEqual(table, [[s1], [s2]]) # test json s = self._get_results_script([job], {f : {}}, export_format='json') self.assertEqual(json.loads(s.run()), [[s2]]) # json export has no header (?)
def test_unicode_excel(self): """Test whether the export can handle unicode in column names and cell values""" try: import openpyxl except ImportError: raise unittest.SkipTest("OpenPyxl not installed, skipping excel test") schema = amcattest.create_test_schema(isarticleschema=True) s1 = u'S1 \xc4\u0193 \u02a2 \u038e\u040e' s2 = u'S2 \u053e\u06a8 \u090c \u0b8f\u0c8a' f = CodingSchemaField.objects.create(codingschema=schema, fieldnr=1, label=s1, fieldtype_id=1, codebook=None) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5) articles = list(job.articleset.articles.all()) amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({f:s2}) # test excel, can't test content but we can test output and no error s = self._get_results_script([job], {f : {}}, export_format='xlsx') self.assertTrue(s.run())
def test_values_table(self): """Does getting a table of values work?""" schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields( ) job = amcattest.create_test_job(unitschema=schema, articleschema=schema) c = amcattest.create_test_coding(codingjob=job) c.update_values({strf: "bla", intf: 1}) self.assertEqual(set(job.values_table().to_list()), {('bla', 1, None)}) code = amcattest.create_test_code(label="CODED") codebook.add_code(code) c2 = amcattest.create_test_coding(codingjob=job) c2.update_values({intf: -1, codef: code}) t = job.values_table() self.assertEqual(set(t.to_list()), {('bla', 1, None), (None, -1, code.id)}) self.assertEqual(set(t.rows), {c, c2})
def test_delete(self): """Simple deletion of a job""" from amcat.models import ArticleSet, Coding s = amcattest.create_test_set(articles=5) j = amcattest.create_test_job(articleset=s) c = amcattest.create_test_coding(codingjob=j) self.assertTrue(CodingJob.objects.filter(pk=j.id).exists()) self.assertTrue(ArticleSet.objects.filter(pk=s.id).exists()) self.assertTrue(Coding.objects.filter(pk=c.id).exists()) DeleteCodingJob(job=j.id).run() self.assertFalse(CodingJob.objects.filter(pk=j.id).exists()) self.assertFalse(ArticleSet.objects.filter(pk=s.id).exists()) self.assertFalse(Coding.objects.filter(pk=c.id).exists())
def test_status(self): """Is initial status 0? Can we set it?""" a = amcattest.create_test_coding() self.assertEqual(a.status.id, 0) self.assertEqual(a.status, CodingStatus.objects.get(pk=STATUS_NOTSTARTED)) a.set_status(STATUS_INPROGRESS) self.assertEqual(a.status, CodingStatus.objects.get(pk=1)) a.set_status(STATUS_COMPLETE) self.assertEqual(a.status, CodingStatus.objects.get(pk=2)) a.set_status(STATUS_IRRELEVANT) self.assertEqual(a.status, CodingStatus.objects.get(pk=9)) a.set_status(STATUS_NOTSTARTED) self.assertEqual(a.status, CodingStatus.objects.get(pk=0))
def test_unicode(self): """Test whether the export can handle unicode in column names and cell values""" schema = amcattest.create_test_schema(isarticleschema=True) s1 = u'S1 \xc4\u0193 \u02a2 \u038e\u040e' s2 = u'S2 \u053e\u06a8 \u090c \u0b8f\u0c8a' f = CodingSchemaField.objects.create(codingschema=schema, fieldnr=1, label=s1, fieldtype_id=1, codebook=None) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5) articles = list(job.articleset.articles.all()) amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({f:s2}) # test csv s = self._get_results_script([job], {f : {}}, export_format='csv') import base64 data = base64.b64decode(s.run()['data']) table = [[cell.decode('utf-8') for cell in row] for row in csv.reader(StringIO(data))] self.assertEqual(table, [[s1], [s2]]) # test json s = self._get_results_script([job], {f : {}}, export_format='json') self.assertEqual(json.loads(s.run()), [[s2]]) # json export has no header (?)
def test_update_value(self): """Does update_value on a codingvalue work?""" a = amcattest.create_test_coding(codingjob=self.job) v = CodingValue.objects.create(coding=a, field=self.intfield, intval=1) v.update_value("99") self.assertEqual(v.value, 99) self.assertRaises(Exception, v.update_value, "abv") v2 = CodingValue.objects.create(coding=a, field=self.codefield, intval=self.c.id) v2.update_value(self.c2) self.assertEqual(v2.value, self.c2) self.assertRaises(Exception, v2.update_value, "abv") self.assertRaises(ValueError, v2.update_value, None)
def test_create_codings(self): """Does get/create coding work?""" a = amcattest.create_test_coding() ca = CodedArticle(a) self.assertEqual(ca.coding, a) self.assertEqual(ca.get_or_create_coding(), a) codingids = set(a.id for a in Coding.objects.all()) ca = CodedArticle(ca.codingjob, amcattest.create_test_article()) self.assertIsNone(ca.coding) a2 = ca.get_or_create_coding() self.assertNotIn(a2.id, codingids) self.assertEqual(ca.coding, a2) self.assertEqual(ca.get_or_create_coding(), a2)
def test_nqueries(self): """Does getting a table of values not use too many queries?""" schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields() job = amcattest.create_test_job(unitschema=schema, articleschema=schema) for i in range(10): c = amcattest.create_test_coding(codingjob=job) c.update_values({strf:"bla %i"%i, intf:i}) job = CodingJob.objects.get(pk=job.id) with self.checkMaxQueries(6): # 1. get schema, 2. get codings, 3. get values, 4. get field, 5+6. get serialiser t = job.values_table() cells = list(t.to_list())
def _add_codings(self, job, codings, narticles=10): """Add codings (list-of-lists) to a job""" s = amcattest.create_test_set(articles=narticles) articles = list(s.articles.all()) fields = list(job.unitschema.fields.all()) result = [] for values in codings: random.shuffle(articles) c = amcattest.create_test_coding(codingjob=job, article=articles[0]) c.update_values(dict(zip(fields, values))) result.append(c) return result # don't use iterator to allow use as statement (eg non-lazy eval)
def test_update_values(self): """Does update_values on an coding work?""" a = amcattest.create_test_coding(codingjob=self.job) self.assertEqual(_valuestr(a), "") a.set_value(self.intfield, 12) self.assertEqual(_valuestr(a), "number:12") a.update_values({self.strfield:"bla"}) self.assertEqual(_valuestr(a), "text:bla") a.update_values({self.strfield:None, self.intfield:"999", self.codefield:self.c}) self.assertEqual(_valuestr(a), "code:CODED;number:999") newfield = CodingSchemaField.objects.create( codingschema=amcattest.create_test_schema(), label="text", fieldtype=self.strfield.fieldtype) self.assertRaises(ValueError, a.update_values, {newfield : "3"})
def setUp(self): """Populate a project with articles, coding etc""" from amcat.models import Language, Article self.project = amcattest.create_test_project() # create a codebook self.codebook = amcattest.create_test_codebook(project=self.project, name="base codebook") en = Language.objects.get(label='en') sv = Language.objects.create(label='sv') _cargs = dict(language=en, codebook=self.codebook) self.code_a = amcattest.create_test_code(label="a", **_cargs) self.code_a.add_label(label=u"\xe5", language=sv) self.code_b = amcattest.create_test_code(label="b", parent=self.code_a, **_cargs) self.code_c = amcattest.create_test_code(label="c", parent=self.code_a, **_cargs) # Codebook bases not yet implemented self.sub_codebook = amcattest.create_test_codebook(project=self.project, name="sub codebook") #self.sub_codebook.add_base(self.codebook) self.code_d = amcattest.create_test_code(label="d", language=en, codebook=self.sub_codebook, parent=self.code_a) CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_c, hide=True) CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_b, parent=None) # create a schema self.schema, _dummy, self.strfield, self.intfield, self.codefield = ( amcattest.create_test_schema_with_fields(project=self.project, codebook=self.sub_codebook)) self.article_hl = u'The great wall of China (\u9577\u57ce)' self.article_text = u"""This is some text with greek characters\n \u03bc\u1fc6\u03bd\u03b9\u03bd \u1f04\u03b5\u03b9\u03b4\u03b5, \u03b8\u03b5\u03ac, \u03a0\u03b7\u03bb\u03b7\u03ca\u03ac\u03b4\u03b5\u03c9 \u1f08\u03c7\u03b9\u03bb\u1fc6\u03bf\u03c2""" self.article = amcattest.create_test_article(headline=self.article_hl, project=self.project, text=self.article_text, date="2012-01-01") self.article = Article.objects.get(pk=self.article.id) # to get date redeserialized self.articleset = amcattest.create_test_set(project=self.project) self.articleset.add(self.article) self.job = amcattest.create_test_job(project=self.project, articleset=self.articleset, unitschema=self.schema, articleschema=self.schema) self.coding = amcattest.create_test_coding(codingjob=self.job, comments="Obvious", article=self.article) self.coding.update_values({self.strfield:"abc", self.intfield:1, self.codefield:self.code_d})
def test_include_uncoded_articles(self): aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True) sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False) cjob = amcattest.create_test_job(2, articleschema=aschema, unitschema=sschema) a1, a2 = cjob.articleset.articles.all() coding = create_test_coding(codingjob=cjob, article=a1) coding.update_values({sstrf: "bla", sintf: 10}) # Default settings should not export uncoded article (a2) fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}} result = self._get_results([cjob], fields, export_level=CODING_LEVEL_BOTH) self.assertEqual(1, len(result)) # Should export extra article if asked to fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}} result = self._get_results([cjob], fields, include_uncoded_articles=True, export_level=CODING_LEVEL_BOTH) self.assertEqual(2, len(result))
def test_include_uncoded_sentences(self): aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True) sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False) a1 = amcattest.create_test_article(text="Zin 1. Zin 2.") a2 = amcattest.create_test_article(text="Zin 1. Zin 2.") aset = amcattest.create_test_set([a1, a2]) cjob = amcattest.create_test_job(articleset=aset, articleschema=aschema, unitschema=sschema) sentence = list(get_or_create_sentences(a1))[1] coding = create_test_coding(codingjob=cjob, article=a1, sentence=sentence) coding.update_values({sstrf: "bla", sintf: 10}) # We expect 1 sentence if we only export codings fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}} result = self._get_results([cjob], fields, include_uncoded_sentences=False, export_level=CODING_LEVEL_BOTH) self.assertEqual(1, len(result)) result = self._get_results([cjob], fields, include_uncoded_sentences=True, export_level=CODING_LEVEL_BOTH) self.assertEqual(3, len(result))
def test_update_values(self): codebook, codes = amcattest.create_test_codebook_with_codes() schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7) articles = list(job.articleset.articles.all()) coding = amcattest.create_test_coding(codingjob=job, article=articles[0]) self.assertEqual(0, coding.values.count()) coding.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id}) self.assertEqual(3, coding.values.count()) self.assertTrue(strf in dict(coding.get_values())) self.assertTrue(intf in dict(coding.get_values())) self.assertTrue(codef in dict(coding.get_values())) self.assertEqual(1, dict(coding.get_values())[intf]) # Does update_values delete values not present in dict? coding.update_values({strf: "blas"}) self.assertEqual(1, coding.values.count()) self.assertTrue(strf in dict(coding.get_values())) self.assertEqual("blas", dict(coding.get_values())[strf])
def test_update_values(self): codebook, codes = amcattest.create_test_codebook_with_codes() schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7) articles = list(job.articleset.articles.all()) coding = amcattest.create_test_coding(codingjob=job, article=articles[0]) self.assertEqual(0, coding.values.count()) coding.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id}) self.assertEqual(3, coding.values.count()) self.assertTrue(strf in dict(coding.get_values())) self.assertTrue(intf in dict(coding.get_values())) self.assertTrue(codef in dict(coding.get_values())) self.assertEqual(1, dict(coding.get_values())[intf]) # Does update_values delete values not present in dict? coding.update_values({strf: "blas"}) self.assertEqual(1, coding.values.count()) self.assertTrue(strf in dict(coding.get_values())) self.assertEqual("blas", dict(coding.get_values())[strf])
def test_update_values(self): """Does update_values on an coding work?""" a = amcattest.create_test_coding(codingjob=self.job) self.assertEqual(_valuestr(a), "") a.set_value(self.intfield, 12) self.assertEqual(_valuestr(a), "number:12") a.update_values({self.strfield: "bla"}) self.assertEqual(_valuestr(a), "text:'bla'") a.update_values({ self.strfield: None, self.intfield: "999", self.codefield: self.c }) self.assertEqual(_valuestr(a), "code:<Code: CODED>;number:999") newfield = CodingSchemaField.objects.create( codingschema=amcattest.create_test_schema(), label="text", fieldtype=self.strfield.fieldtype) self.assertRaises(ValueError, a.update_values, {newfield: "3"})
def test_unicode_excel(self): """Test whether the export can handle unicode in column names and cell values""" try: import openpyxl except ImportError: raise unittest.SkipTest("OpenPyxl not installed, skipping excel test") schema = amcattest.create_test_schema(isarticleschema=True) s1 = u'S1 \xc4\u0193 \u02a2 \u038e\u040e' s2 = u'S2 \u053e\u06a8 \u090c \u0b8f\u0c8a' f = CodingSchemaField.objects.create(codingschema=schema, fieldnr=1, label=s1, fieldtype_id=1, codebook=None) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5) articles = list(job.articleset.articles.all()) coding = amcattest.create_test_coding(codingjob=job, article=articles[0]) coding.update_values({f:s2}) # test excel, can't test content but we can test output and no error s = self._get_results_script([job], {f : {}}, export_format='xlsx') self.assertTrue(s.run())
def setUp(self): self.s1 = amcattest.create_test_set(5) self.a = list(self.s1.articles.all().order_by('id')) self.ids = [a.id for a in self.a] self.m1 = "Telegraaf" self.m2 = "NRC" self.m3 = "AD" self.a[0].set_property("medium", self.m1) self.a[1].set_property("medium", self.m2) self.a[2].set_property("medium", self.m2) self.a[3].set_property("medium", self.m3) self.a[4].set_property("medium", self.m3) self.a[0].text = "aap." self.a[1].text = "aap. noot." self.a[2].text = "aap. noot. mies." self.a[0].date = datetime.datetime(2015, 1, 1) self.a[1].date = datetime.datetime(2015, 1, 1) self.a[2].date = datetime.datetime(2015, 2, 1) self.a[3].date = datetime.datetime(2016, 1, 1) self.a[4].date = datetime.datetime(2016, 1, 1) self.a[0].save() self.a[1].save() self.a[2].save() self.a[3].save() self.a[4].save() # Uncomment if ever using elastic :) # self.s1.refresh_index(full_refresh=True) self.schema, self.codebook, self.strf, self.intf, self.codef, self.boolf, self.qualf = ( amcattest.create_test_schema_with_fields(isarticleschema=True)) self.sschema, self.scodebook, self.sstrf, self.sintf, self.scodef, self.sboolf, self.squalf = ( amcattest.create_test_schema_with_fields(isarticleschema=False)) # Article self.codes = self.codebook.get_codes() self.code_A, = [c for c in self.codes if c.label == "A"] self.code_B, = [c for c in self.codes if c.label == "B"] self.code_A1, = [c for c in self.codes if c.label == "A1"] # Sentence self.scodes = self.codebook.get_codes() self.scode_A, = [c for c in self.scodes if c.label == "A"] self.scode_B, = [c for c in self.scodes if c.label == "B"] self.scode_A1, = [c for c in self.scodes if c.label == "A1"] self.scode_A1b, = [c for c in self.scodes if c.label == "A1b"] # Does not get fired in unit test? for article in [self.a[0], self.a[1], self.a[2], self.a[3], self.a[4]]: get_or_create_sentences(article) self.job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema, unitschema=self.sschema) self.c1 = amcattest.create_test_coding(codingjob=self.job, article=self.a[0]) self.c1.update_values({self.codef: self.code_A.id, self.intf: 4, self.qualf: 4}) self.c2 = amcattest.create_test_coding(codingjob=self.job, article=self.a[1]) self.c2.update_values({self.codef: self.code_A.id, self.intf: 2, self.qualf: 1}) self.c3 = amcattest.create_test_coding(codingjob=self.job, article=self.a[2]) self.c3.update_values({self.codef: self.code_B.id, self.intf: 1, self.qualf: 2}) self.c4 = amcattest.create_test_coding(codingjob=self.job, article=self.a[3]) self.c4.update_values({self.codef: self.code_A1.id, self.intf: 1}) self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a[0], sentence=self.a[0].sentences.all()[0]) self.sentence_coding.update_values({self.scodef: self.scode_A1.id, self.sintf: 1}) self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a[2], sentence=self.a[2].sentences.all()[0]) self.sentence_coding.update_values({self.scodef: self.scode_A1.id, self.sintf: 1}) self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a[2], sentence=self.a[2].sentences.all()[0]) self.sentence_coding.update_values({self.scodef: self.scode_A1b.id, self.sintf: 1}) # Try to confuse aggregator by inserting multiple codingjobs job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema) c4 = amcattest.create_test_coding(codingjob=job, article=self.a[2]) c4.update_values({self.codef: self.code_B.id, self.intf: 10, self.qualf: 8})
def test_nqueries(self): from amcat.tools import amcatlogging amcatlogging.setup() codebook, codes = amcattest.create_test_codebook_with_codes() schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields( codebook=codebook) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7) articles = list(job.articleset.articles.all()) amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"] }) amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"] }) amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"] }) amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"] }) amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"] }) codingjobs = list(CodingJob.objects.filter(pk__in=[job.id])) c = codingjobs[0].codings.all()[0] amcatlogging.debug_module('django.db.backends') script = self._get_results_script([job], {strf: {}, intf: {}}) with self.checkMaxQueries(5): list(csv.reader(StringIO(script.run()))) script = self._get_results_script([job], { strf: {}, intf: {}, codef: dict(ids=True) }) with self.checkMaxQueries(5): list(csv.reader(StringIO(script.run()))) script = self._get_results_script([job], { strf: {}, intf: {}, codef: dict(labels=True) }) with self.checkMaxQueries(5): list(csv.reader(StringIO(script.run())))
def setUp(self): """Populate a project with articles, coding etc""" from amcat.models import Language, Article self.project = amcattest.create_test_project() # create a codebook self.codebook = amcattest.create_test_codebook(project=self.project, name="base codebook") en = Language.objects.get(label='en') sv = Language.objects.create(label='sv') _cargs = dict(language=en, codebook=self.codebook) self.code_a = amcattest.create_test_code(label="a", **_cargs) self.code_a.add_label(label=u"\xe5", language=sv) self.code_b = amcattest.create_test_code(label="b", parent=self.code_a, **_cargs) self.code_c = amcattest.create_test_code(label="c", parent=self.code_a, **_cargs) # Codebook bases not yet implemented self.sub_codebook = amcattest.create_test_codebook( project=self.project, name="sub codebook") #self.sub_codebook.add_base(self.codebook) self.code_d = amcattest.create_test_code(label="d", language=en, codebook=self.sub_codebook, parent=self.code_a) CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_c, hide=True) CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_b, parent=None) # create a schema self.schema, _dummy, self.strfield, self.intfield, self.codefield = ( amcattest.create_test_schema_with_fields( project=self.project, codebook=self.sub_codebook)) self.article_hl = u'The great wall of China (\u9577\u57ce)' self.article_text = u"""This is some text with greek characters\n \u03bc\u1fc6\u03bd\u03b9\u03bd \u1f04\u03b5\u03b9\u03b4\u03b5, \u03b8\u03b5\u03ac, \u03a0\u03b7\u03bb\u03b7\u03ca\u03ac\u03b4\u03b5\u03c9 \u1f08\u03c7\u03b9\u03bb\u1fc6\u03bf\u03c2""" self.article = amcattest.create_test_article(headline=self.article_hl, project=self.project, text=self.article_text, date="2012-01-01") self.article = Article.objects.get( pk=self.article.id) # to get date redeserialized self.articleset = amcattest.create_test_set(project=self.project) self.articleset.add(self.article) self.job = amcattest.create_test_job(project=self.project, articleset=self.articleset, unitschema=self.schema, articleschema=self.schema) self.coding = amcattest.create_test_coding(codingjob=self.job, comments="Obvious", article=self.article) self.coding.update_values({ self.strfield: "abc", self.intfield: 1, self.codefield: self.code_d })
def setUp(self): self.s1 = amcattest.create_test_set(5) self.a1, self.a2, self.a3, self.a4, self.a5 = self.s1.articles.all() self.m1 = "Telegraaf" self.m2 = "NRC" self.m3 = "AD" self.a1.set_property("medium", self.m1) self.a2.set_property("medium", self.m2) self.a3.set_property("medium", self.m2) self.a4.set_property("medium", self.m3) self.a5.set_property("medium", self.m3) self.a1.text = "aap." self.a2.text = "aap. noot." self.a3.text = "aap. noot. mies." self.a1.date = datetime.datetime(2015, 1, 1) self.a2.date = datetime.datetime(2015, 1, 1) self.a3.date = datetime.datetime(2015, 2, 1) self.a4.date = datetime.datetime(2016, 1, 1) self.a5.date = datetime.datetime(2016, 1, 1) self.a1.save() self.a2.save() self.a3.save() self.a4.save() self.a5.save() # Uncomment if ever using elastic :) # self.s1.refresh_index(full_refresh=True) self.schema, self.codebook, self.strf, self.intf, self.codef, self.boolf, self.qualf = ( amcattest.create_test_schema_with_fields(isarticleschema=True)) self.sschema, self.scodebook, self.sstrf, self.sintf, self.scodef, self.sboolf, self.squalf = ( amcattest.create_test_schema_with_fields(isarticleschema=False)) # Article self.codes = self.codebook.get_codes() self.code_A, = [c for c in self.codes if c.label == "A"] self.code_B, = [c for c in self.codes if c.label == "B"] self.code_A1, = [c for c in self.codes if c.label == "A1"] # Sentence self.scodes = self.codebook.get_codes() self.scode_A, = [c for c in self.scodes if c.label == "A"] self.scode_B, = [c for c in self.scodes if c.label == "B"] self.scode_A1, = [c for c in self.scodes if c.label == "A1"] # Does not get fired in unit test? for article in [self.a1, self.a2, self.a3, self.a4, self.a5]: get_or_create_sentences(article) self.job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema, unitschema=self.sschema) self.c1 = amcattest.create_test_coding(codingjob=self.job, article=self.a1) self.c1.update_values({ self.codef: self.code_A.id, self.intf: 4, self.qualf: 4 }) self.c2 = amcattest.create_test_coding(codingjob=self.job, article=self.a2) self.c2.update_values({ self.codef: self.code_A.id, self.intf: 2, self.qualf: 1 }) self.c3 = amcattest.create_test_coding(codingjob=self.job, article=self.a3) self.c3.update_values({ self.codef: self.code_B.id, self.intf: 1, self.qualf: 2 }) self.c4 = amcattest.create_test_coding(codingjob=self.job, article=self.a4) self.c4.update_values({self.codef: self.code_A1.id, self.intf: 1}) self.sentence_coding = amcattest.create_test_coding( codingjob=self.job, article=self.a1, sentence=self.a1.sentences.all()[0]) self.sentence_coding.update_values({ self.scodef: self.scode_A1.id, self.sintf: 1 }) # Try to confuse aggregator by inserting multiple codingjobs job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema) c4 = amcattest.create_test_coding(codingjob=job, article=self.a3) c4.update_values({ self.codef: self.code_B.id, self.intf: 10, self.qualf: 8 })
def test_get_rows(self): schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields( ) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5) articles = list(job.articleset.articles.all()) c = amcattest.create_test_coding(codingjob=job, article=articles[0]) ca = job.get_coded_article(articles[0]) # simple coding rows = set( _get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], None, c, None)}) # test uncoded_articles rows = set( _get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=True)) self.assertEqual( rows, {(job, ca, articles[0], None, c, None)} | {(job, job.get_coded_article(a), a, None, None, None) for a in articles[1:]}) # test sentence s = amcattest.create_test_sentence(article=articles[0]) sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s) rows = set( _get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], None, c, None)}) rows = set( _get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], s, c, sc)}) # multiple sentence codings on the same article should duplicate article(coding) s2 = amcattest.create_test_sentence(article=articles[0]) sc2 = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s2) rows = set( _get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False)) self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2)}) # if an article contains an article coding but no sentence coding, it should still show up with sentence=True c2 = amcattest.create_test_coding(codingjob=job, article=articles[1]) rows = set( _get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False)) self.assertEqual( rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2), (job, job.get_coded_article( articles[1]), articles[1], None, c2, None)})
def test_nqueries(self): codebook, codes = amcattest.create_test_codebook_with_codes() schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields( codebook=codebook) job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7) articles = list(job.articleset.articles.all()) log.info(codes) amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"].id }) amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"].id }) amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"].id }) amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"].id }) amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({ strf: "bla", intf: 1, codef: codes["A1b"].id }) codingjobs = list(CodingJob.objects.filter(pk__in=[job.id])) c = list(codingjobs[0].codings)[0] script = self._get_results_script([job], {strf: {}, intf: {}}) with self.checkMaxQueries(9): list(csv.reader(StringIO(script.run()))) script = self._get_results_script([job], { strf: {}, intf: {}, codef: dict(ids=True) }) with self.checkMaxQueries(9): list(csv.reader(StringIO(script.run()))) script = self._get_results_script([job], { strf: {}, intf: {}, codef: dict(labels=True) }) with self.checkMaxQueries(9): list(csv.reader(StringIO(script.run())))
def setUp(self): self.s1 = amcattest.create_test_set(5) self.a1, self.a2, self.a3, self.a4, self.a5 = self.s1.articles.all() self.m1 = self.a1.medium self.m2 = self.a2.medium self.m3 = self.a3.medium self.m4 = self.a4.medium self.a3.medium = self.m2 self.a3.save() self.a1.text = "aap." self.a2.text = "aap. noot." self.a3.text = "aap. noot. mies." self.a1.date = datetime.datetime(2015, 0o1, 0o1) self.a2.date = datetime.datetime(2015, 0o1, 0o1) self.a3.date = datetime.datetime(2015, 0o2, 0o1) self.a4.date = datetime.datetime(2016, 0o1, 0o1) self.a5.date = datetime.datetime(2016, 0o1, 0o1) self.a1.save() self.a2.save() self.a3.save() self.a4.save() self.a5.save() # Uncomment if ever using elastic :) # self.s1.refresh_index(full_refresh=True) self.schema, self.codebook, self.strf, self.intf, self.codef, self.boolf, self.qualf = ( amcattest.create_test_schema_with_fields(isarticleschema=True)) self.sschema, self.scodebook, self.sstrf, self.sintf, self.scodef, self.sboolf, self.squalf = ( amcattest.create_test_schema_with_fields(isarticleschema=False)) # Article self.codes = self.codebook.get_codes() self.code_A, = [c for c in self.codes if c.label == "A"] self.code_B, = [c for c in self.codes if c.label == "B"] self.code_A1, = [c for c in self.codes if c.label == "A1"] # Sentence self.scodes = self.codebook.get_codes() self.scode_A, = [c for c in self.scodes if c.label == "A"] self.scode_B, = [c for c in self.scodes if c.label == "B"] self.scode_A1, = [c for c in self.scodes if c.label == "A1"] # Does not get fired in unit test? for article in [self.a1, self.a2, self.a3, self.a4, self.a5]: get_or_create_sentences(article) self.job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema, unitschema=self.sschema) self.c1 = amcattest.create_test_coding(codingjob=self.job, article=self.a1) self.c1.update_values({self.codef: self.code_A.id, self.intf: 4, self.qualf: 4}) self.c2 = amcattest.create_test_coding(codingjob=self.job, article=self.a2) self.c2.update_values({self.codef: self.code_A.id, self.intf: 2, self.qualf: 1}) self.c3 = amcattest.create_test_coding(codingjob=self.job, article=self.a3) self.c3.update_values({self.codef: self.code_B.id, self.intf: 1, self.qualf: 2}) self.c4 = amcattest.create_test_coding(codingjob=self.job, article=self.a4) self.c4.update_values({self.codef: self.code_A1.id, self.intf: 1}) self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a1, sentence=self.a1.sentences.all()[0]) self.sentence_coding.update_values({self.scodef: self.scode_A1.id, self.sintf: 1}) # Try to confuse aggregator by inserting multiple codingjobs job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema) c4 = amcattest.create_test_coding(codingjob=job, article=self.a3) c4.update_values({self.codef: self.code_B.id, self.intf: 10, self.qualf: 8})