def test_nqueries_sentence_codings(self):
        aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(
            isarticleschema=True)
        sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(
            isarticleschema=False)
        cjob = amcattest.create_test_job(10,
                                         articleschema=aschema,
                                         unitschema=sschema)

        for article in cjob.articleset.articles.all():
            coding = create_test_coding(codingjob=cjob, article=article)
            coding.update_values({astrf: "blas", aintf: 20})
            for sentence in get_or_create_sentences(article):
                coding = create_test_coding(codingjob=cjob,
                                            article=article,
                                            sentence=sentence)
                coding.update_values({sstrf: "bla", sintf: 10})

        fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
        script = self._get_results_script([cjob],
                                          fields,
                                          export_level=CODING_LEVEL_BOTH)

        with self.checkMaxQueries(9):
            list(csv.reader(StringIO(script.run())))
示例#2
0
    def test_results(self):
        codebook, codes = amcattest.create_test_codebook_with_codes()
        schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook,
                                                                                       isarticleschema=True)
        sschema, codebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
        job = amcattest.create_test_job(unitschema=sschema, articleschema=schema, narticles=5)
        articles = list(job.articleset.articles.all())

        c = amcattest.create_test_coding(codingjob=job, article=articles[0])

        # test simple coding with a codebook code
        c.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
        self.assertEqual(self._get_results([job], {strf: {}, intf: {}, codef: dict(ids=True)}),
                         [('bla', 1, codes["A1b"].id)])
        # test multiple codings and parents
        c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
        c2.update_values({strf: "blx", intf: 1, codef: codes["B1"].id})
        self.assertEqual(set(self._get_results([job], {strf: {}, intf: {}, codef: dict(labels=True, parents=2)})),
                         {('bla', 1, "A", "A1", "A1b"), ('blx', 1, "B", "B1", "B1")})


        # test sentence result
        s = amcattest.create_test_sentence(article=articles[0])
        sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
        sc.update_values({sstrf: "z", sintf: -1, scodef: codes["A"].id})

        self.assertEqual(set(self._get_results([job], {strf: {}, sstrf: {}, sintf: {}}, export_level=2)),
                         {('bla', 'z', -1), ('blx', None, None)})
    def test_results(self):
        codebook, codes = amcattest.create_test_codebook_with_codes()
        schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook, isarticleschema=True)
        sschema, codebook, sstrf, sintf, scodef = amcattest.create_test_schema_with_fields(codebook=codebook)
        job = amcattest.create_test_job(unitschema=sschema, articleschema=schema, narticles=5)
        articles = list(job.articleset.articles.all())

        c = amcattest.create_test_coding(codingjob=job, article=articles[0])

        # test simple coding with a codebook code
        c.update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
        self.assertEqual(self._get_results([job], {strf : {}, intf : {}, codef : dict(ids=True)}),
                         [('bla', 1, codes["A1b"].id)])
        # test multiple codings and parents
        c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
        c2.update_values({strf:"blx", intf:1, codef:codes["B1"].id})
        self.assertEqual(set(self._get_results([job], {strf : {}, intf : {}, codef : dict(labels=True, parents=2)})),
                         {('bla', 1, "A", "A1", "A1b"), ('blx', 1, "B", "B1", "B1")})


        # test sentence result
        s = amcattest.create_test_sentence(article=articles[0])
        sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
        sc.update_values({sstrf:"z", sintf:-1, scodef:codes["A"].id})

        self.assertEqual(set(self._get_results([job], {strf : {}, sstrf : {}, sintf : {}}, export_level=2)),
                         {('bla', 'z', -1), ('blx', None, None)})
    def test_include_uncoded_sentences(self):
        aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(
            isarticleschema=True)
        sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(
            isarticleschema=False)
        a1 = amcattest.create_test_article(text="Zin 1. Zin 2.")
        a2 = amcattest.create_test_article(text="Zin 1. Zin 2.")
        aset = amcattest.create_test_set([a1, a2])
        cjob = amcattest.create_test_job(articleset=aset,
                                         articleschema=aschema,
                                         unitschema=sschema)

        sentence = list(get_or_create_sentences(a1))[1]
        coding = create_test_coding(codingjob=cjob,
                                    article=a1,
                                    sentence=sentence)
        coding.update_values({sstrf: "bla", sintf: 10})

        # We expect 1 sentence if we only export codings
        fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
        result = self._get_results([cjob],
                                   fields,
                                   include_uncoded_sentences=False,
                                   export_level=CODING_LEVEL_BOTH)
        self.assertEqual(1, len(result))

        result = self._get_results([cjob],
                                   fields,
                                   include_uncoded_sentences=True,
                                   export_level=CODING_LEVEL_BOTH)
        self.assertEqual(3, len(result))
    def test_include_uncoded_articles(self):
        aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(
            isarticleschema=True)
        sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(
            isarticleschema=False)
        cjob = amcattest.create_test_job(2,
                                         articleschema=aschema,
                                         unitschema=sschema)
        a1, a2 = cjob.articleset.articles.all()
        coding = create_test_coding(codingjob=cjob, article=a1)
        coding.update_values({sstrf: "bla", sintf: 10})

        # Default settings should not export uncoded article (a2)
        fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
        result = self._get_results([cjob],
                                   fields,
                                   export_level=CODING_LEVEL_BOTH)
        self.assertEqual(1, len(result))

        # Should export extra article if asked to
        fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
        result = self._get_results([cjob],
                                   fields,
                                   include_uncoded_articles=True,
                                   export_level=CODING_LEVEL_BOTH)
        self.assertEqual(2, len(result))
    def test_replace_codings(self):
        schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(isarticleschema=True)
        schema2, codebook2, strf2, intf2, codef2 = amcattest.create_test_schema_with_fields(isarticleschema=True)
        codingjob = amcattest.create_test_job(articleschema=schema, narticles=10)

        coded_article = CodedArticle.objects.get(article=codingjob.articleset.articles.all()[0], codingjob=codingjob)
        coded_article.replace_codings([self._get_coding_dict(intval=10, field_id=codef.id)])

        self.assertEqual(1, coded_article.codings.all().count())
        self.assertEqual(1, coded_article.codings.all()[0].values.all().count())
        coding = coded_article.codings.all()[0]
        value = coding.values.all()[0]
        self.assertEqual(coding.sentence, None)
        self.assertEqual(value.strval, None)
        self.assertEqual(value.intval, 10)
        self.assertEqual(value.field, codef)

        # Overwrite previous coding
        coded_article.replace_codings([self._get_coding_dict(intval=11, field_id=intf.id)])
        self.assertEqual(1, coded_article.codings.all().count())
        self.assertEqual(1, coded_article.codings.all()[0].values.all().count())
        coding = coded_article.codings.all()[0]
        value = coding.values.all()[0]
        self.assertEqual(coding.sentence, None)
        self.assertEqual(value.strval, None)
        self.assertEqual(value.intval, 11)
        self.assertEqual(value.field, intf)

        # Try to insert illigal values
        illval1 = self._get_coding_dict(intval=1, strval="a", field_id=intf.id)
        illval2 = self._get_coding_dict(field_id=intf.id)
        illval3 = self._get_coding_dict(intval=1)
        illval4 = self._get_coding_dict(intval=1, field_id=strf2.id)

        self.assertRaises(ValueError, coded_article.replace_codings, [illval1])
        self.assertRaises(ValueError, coded_article.replace_codings, [illval2])
        self.assertRaises(IntegrityError, coded_article.replace_codings, [illval3])
        self.assertRaises(ValueError, coded_article.replace_codings, [illval4])

        # Unspecified values default to None
        val = self._get_coding_dict(intval=1, field_id=intf.id)
        del val["values"][0]["strval"]
        coded_article.replace_codings([val])
        value = coded_article.codings.all()[0].values.all()[0]
        self.assertEqual(value.strval, None)
        self.assertEqual(value.intval, 1)

        val = self._get_coding_dict(strval="a", field_id=intf.id)
        del val["values"][0]["intval"]
        coded_article.replace_codings([val])
        value = coded_article.codings.all()[0].values.all()[0]
        self.assertEqual(value.strval, "a")
        self.assertEqual(value.intval, None)
    def test_nqueries(self):
        from amcat.tools import amcatlogging
        amcatlogging.setup()

        codebook, codes = amcattest.create_test_codebook_with_codes()
        schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
        job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
        articles = list(job.articleset.articles.all())

        log.info(codes)
        amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
        amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
        amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
        amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
        amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})

        codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
        c = list(codingjobs[0].codings)[0]
        amcatlogging.debug_module('django.db.backends')

        script = self._get_results_script([job], {strf : {}, intf : {}})
        with self.checkMaxQueries(8):
            list(csv.reader(StringIO(script.run())))


        script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(ids=True)})
        with self.checkMaxQueries(8):
            list(csv.reader(StringIO(script.run())))


        script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(labels=True)})
        with self.checkMaxQueries(8):
            list(csv.reader(StringIO(script.run())))
 def test_get_rows(self):
     schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields()
     job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5)
     articles = list(job.articleset.articles.all())
     c = amcattest.create_test_coding(codingjob=job, article=articles[0])
     ca = job.get_coded_article(articles[0])
     # simple coding
     rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], None, c, None)})
     # test uncoded_articles
     rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=True))
     self.assertEqual(rows, {(job, ca, articles[0], None, c, None)} | {(job, job.get_coded_article(a), a, None, None, None)
                                                                       for a in articles[1:]})
     # test sentence
     s = amcattest.create_test_sentence(article=articles[0])
     sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
     rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], None, c, None)})
     rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], s, c, sc)})
     # multiple sentence codings on the same article should duplicate article(coding)
     s2 = amcattest.create_test_sentence(article=articles[0])
     sc2 = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s2)
     rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2)})
     # if an article contains an article coding but no sentence coding, it should still show up with sentence=True
     c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
     rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2),
                             (job, job.get_coded_article(articles[1]), articles[1], None, c2, None)})
示例#9
0
    def test_to_json(self):
        import functools

        o1, o2 = amcattest.create_test_code(), amcattest.create_test_code()
        schema_with_fields = amcattest.create_test_schema_with_fields()
        code_field = schema_with_fields[4]
        c = functools.partial(self.condition, schema_with_fields[0])

        tree = to_json(parse(c("{}=={}".format(code_field.id, o1.id))))
        self.assertEquals(
            json.loads(tree), {
                "type":
                EQUALS,
                "values": [{
                    "type": "codingschemafield",
                    "id": code_field.id
                }, {
                    "type": "code",
                    "id": o1.id
                }]
            })

        tree = parse(c("{}=={}".format(code_field.id, o1.id)))
        self.assertEquals(json.dumps(to_json(tree, serialise=False)),
                          to_json(tree))
示例#10
0
 def test_get_rows(self):
     schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields()
     job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=5)
     articles = list(job.articleset.articles.all())
     c = amcattest.create_test_coding(codingjob=job, article=articles[0])
     ca = job.get_coded_article(articles[0])
     # simple coding
     rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], None, c, None)})
     # test uncoded_articles
     rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=True))
     self.assertEqual(rows,
                      {(job, ca, articles[0], None, c, None)} | {(job, job.get_coded_article(a), a, None, None, None)
                                                                 for a in articles[1:]})
     # test sentence
     s = amcattest.create_test_sentence(article=articles[0])
     sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
     rows = set(_get_rows([job], include_sentences=False, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], None, c, None)})
     rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], s, c, sc)})
     # multiple sentence codings on the same article should duplicate article(coding)
     s2 = amcattest.create_test_sentence(article=articles[0])
     sc2 = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s2)
     rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2)})
     # if an article contains an article coding but no sentence coding, it should still show up with sentence=True
     c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
     rows = set(_get_rows([job], include_sentences=True, include_multiple=True, include_uncoded_articles=False))
     self.assertEqual(rows, {(job, ca, articles[0], s, c, sc), (job, ca, articles[0], s2, c, sc2),
                             (job, job.get_coded_article(articles[1]), articles[1], None, c2, None)})
    def test_nqueries(self):
        from amcat.tools import amcatlogging
        amcatlogging.setup()

        codebook, codes = amcattest.create_test_codebook_with_codes()
        schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
        job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
        articles = list(job.articleset.articles.all())
        
        amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
        amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
        amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
        amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
        amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})                        

        codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
        c = codingjobs[0].codings.all()[0]
        amcatlogging.debug_module('django.db.backends')

        script = self._get_results_script([job], {strf : {}, intf : {}})
        with self.checkMaxQueries(5):
            list(csv.reader(StringIO(script.run())))


        script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(ids=True)})
        with self.checkMaxQueries(5):
            list(csv.reader(StringIO(script.run())))


        script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(labels=True)})
        with self.checkMaxQueries(5):
            list(csv.reader(StringIO(script.run())))
示例#12
0
    def test_nqueries(self):
        codebook, codes = amcattest.create_test_codebook_with_codes()
        schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
        job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
        articles = list(job.articleset.articles.all())

        log.info(codes)
        amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values(
            {strf: "bla", intf: 1, codef: codes["A1b"].id})
        amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values(
            {strf: "bla", intf: 1, codef: codes["A1b"].id})
        amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values(
            {strf: "bla", intf: 1, codef: codes["A1b"].id})
        amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values(
            {strf: "bla", intf: 1, codef: codes["A1b"].id})
        amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values(
            {strf: "bla", intf: 1, codef: codes["A1b"].id})

        codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
        c = list(codingjobs[0].codings)[0]

        script = self._get_results_script([job], {strf: {}, intf: {}})
        with self.checkMaxQueries(9):
            list(csv.reader(StringIO(script.run())))

        script = self._get_results_script([job], {strf: {}, intf: {}, codef: dict(ids=True)})
        with self.checkMaxQueries(9):
            list(csv.reader(StringIO(script.run())))

        script = self._get_results_script([job], {strf: {}, intf: {}, codef: dict(labels=True)})
        with self.checkMaxQueries(9):
            list(csv.reader(StringIO(script.run())))
示例#13
0
    def test_include_uncoded_articles(self):
        aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True)
        sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False)
        cjob = amcattest.create_test_job(2, articleschema=aschema, unitschema=sschema)
        a1, a2 = cjob.articleset.articles.all()
        coding = create_test_coding(codingjob=cjob, article=a1)
        coding.update_values({sstrf: "bla", sintf: 10})

        # Default settings should not export uncoded article (a2)
        fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
        result = self._get_results([cjob], fields, export_level=CODING_LEVEL_BOTH)
        self.assertEqual(1, len(result))

        # Should export extra article if asked to
        fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
        result = self._get_results([cjob], fields, include_uncoded_articles=True, export_level=CODING_LEVEL_BOTH)
        self.assertEqual(2, len(result))
示例#14
0
    def test_nqueries_sentence_codings(self):
        aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True)
        sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False)
        cjob = amcattest.create_test_job(10, articleschema=aschema, unitschema=sschema)

        for article in cjob.articleset.articles.all():
            coding = create_test_coding(codingjob=cjob, article=article)
            coding.update_values({astrf: "blas", aintf: 20})
            for sentence in get_or_create_sentences(article):
                coding = create_test_coding(codingjob=cjob, article=article, sentence=sentence)
                coding.update_values({sstrf: "bla", sintf: 10})

        fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
        script = self._get_results_script([cjob], fields, export_level=CODING_LEVEL_BOTH)

        with self.checkMaxQueries(9):
            list(csv.reader(StringIO(script.run())))
    def test_clean_tree(self):
        import functools

        o1, o2 = amcattest.create_test_code(), amcattest.create_test_code()
        codebook, codebook_codes = amcattest.create_test_codebook_with_codes()
        schema_with_fields = amcattest.create_test_schema_with_fields(codebook=codebook)

        schema = schema_with_fields[0]
        code_field = schema_with_fields[4]

        c = functools.partial(self.condition, schema)

        tree = parse(c("{code_field.id}=={o1.id}".format(**locals())))
        self.assertRaises(ValidationError, clean_tree, schema, tree)
        tree = parse(c("{code_field.id}=={code.id}".format(code_field=code_field, code=codebook.codes[0])))
        self.assertEquals(clean_tree(schema, tree), None)
        self.assertRaises(ValidationError, clean_tree, amcattest.create_test_schema_with_fields()[0], tree)
示例#16
0
    def test_include_uncoded_sentences(self):
        aschema, acodebook, astrf, aintf, acodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=True)
        sschema, scodebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(isarticleschema=False)
        a1 = amcattest.create_test_article(text="Zin 1. Zin 2.")
        a2 = amcattest.create_test_article(text="Zin 1. Zin 2.")
        aset = amcattest.create_test_set([a1, a2])
        cjob = amcattest.create_test_job(articleset=aset, articleschema=aschema, unitschema=sschema)

        sentence = list(get_or_create_sentences(a1))[1]
        coding = create_test_coding(codingjob=cjob, article=a1, sentence=sentence)
        coding.update_values({sstrf: "bla", sintf: 10})

        # We expect 1 sentence if we only export codings
        fields = {sstrf: {}, sintf: {}, astrf: {}, aintf: {}}
        result = self._get_results([cjob], fields, include_uncoded_sentences=False, export_level=CODING_LEVEL_BOTH)
        self.assertEqual(1, len(result))

        result = self._get_results([cjob], fields, include_uncoded_sentences=True, export_level=CODING_LEVEL_BOTH)
        self.assertEqual(3, len(result))
示例#17
0
    def test_clean_tree(self):
        import functools

        o1, o2 = amcattest.create_test_code(), amcattest.create_test_code()
        codebook, codebook_codes = amcattest.create_test_codebook_with_codes()
        schema_with_fields = amcattest.create_test_schema_with_fields(
            codebook=codebook)

        schema = schema_with_fields[0]
        code_field = schema_with_fields[4]

        c = functools.partial(self.condition, schema)

        tree = parse(c("{code_field.id}=={o1.id}".format(**locals())))
        self.assertRaises(ValidationError, clean_tree, schema, tree)
        tree = parse(
            c("{code_field.id}=={code.id}".format(code_field=code_field,
                                                  code=codebook.codes[0])))
        self.assertEquals(clean_tree(schema, tree), None)
        self.assertRaises(ValidationError, clean_tree,
                          amcattest.create_test_schema_with_fields()[0], tree)
示例#18
0
    def setUp(self):
        """Set up a simple coding schema with fields to use for testing"""
        super(TestCoding, self).setUp()

        self.schema, self.codebook,  self.strfield, self.intfield, self.codefield = (
            amcattest.create_test_schema_with_fields())

        self.c = amcattest.create_test_code(label="CODED")
        self.c2 = amcattest.create_test_code(label="CODE2")
        self.codebook.add_code(self.c)
        self.codebook.add_code(self.c2)
        
        self.job = amcattest.create_test_job(unitschema=self.schema, articleschema=self.schema)
示例#19
0
    def setUp(self):
        """Set up a simple codingschema with fields to use for testing"""
        super(TestCoding, self).setUp()

        self.schema, self.codebook, self.strfield, self.intfield, self.codefield, _, _ = (
            amcattest.create_test_schema_with_fields())

        self.c = amcattest.create_test_code(label="CODED")
        self.c2 = amcattest.create_test_code(label="CODE2")
        self.codebook.add_code(self.c)
        self.codebook.add_code(self.c2)

        self.job = amcattest.create_test_job(unitschema=self.schema, articleschema=self.schema)
示例#20
0
    def test_nqueries(self):
        """Does getting a table of values not use too many queries?"""
        
        schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields()
        job = amcattest.create_test_job(unitschema=schema, articleschema=schema)

        for i in range(10):
            c = amcattest.create_test_coding(codingjob=job)
            c.update_values({strf:"bla %i"%i, intf:i})

        job = CodingJob.objects.get(pk=job.id)
        with self.checkMaxQueries(6):
            # 1. get schema, 2. get codings, 3. get values, 4. get field, 5+6. get serialiser
            t = job.values_table()
            cells = list(t.to_list())
    def test_to_json(self):
        import functools

        o1, o2 = amcattest.create_test_code(), amcattest.create_test_code()
        schema_with_fields = amcattest.create_test_schema_with_fields()
        code_field = schema_with_fields[4]
        c = functools.partial(self.condition, schema_with_fields[0])

        tree = to_json(parse(c("{}=={}".format(code_field.id, o1.id))))
        self.assertEquals(json.loads(tree), {"type":EQUALS, "values":[
            {"type":"codingschemafield", "id" : code_field.id},
            {"type":"code", "id" : o1.id}
        ]})

        tree = parse(c("{}=={}".format(code_field.id, o1.id)))
        self.assertEquals(json.dumps(to_json(tree, serialise=False)), to_json(tree))
示例#22
0
    def setUp(self):
        """Populate a project with articles, coding etc"""
        from amcat.models import Language, Article
        
        self.project = amcattest.create_test_project()

        # create a codebook
        self.codebook = amcattest.create_test_codebook(project=self.project, name="base codebook")
        en = Language.objects.get(label='en')    
        sv = Language.objects.create(label='sv')
        _cargs = dict(language=en, codebook=self.codebook)
        self.code_a = amcattest.create_test_code(label="a", **_cargs)
        self.code_a.add_label(label=u"\xe5", language=sv)
        self.code_b = amcattest.create_test_code(label="b", parent=self.code_a, **_cargs)
        self.code_c = amcattest.create_test_code(label="c", parent=self.code_a, **_cargs)

        # Codebook bases not yet implemented
        self.sub_codebook = amcattest.create_test_codebook(project=self.project, name="sub codebook")
        #self.sub_codebook.add_base(self.codebook)
        self.code_d = amcattest.create_test_code(label="d", language=en,
                                                 codebook=self.sub_codebook, parent=self.code_a)
        CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_c, hide=True)
        CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_b, parent=None)

        # create a schema
        self.schema, _dummy, self.strfield, self.intfield, self.codefield = (
            amcattest.create_test_schema_with_fields(project=self.project, codebook=self.sub_codebook))

        self.article_hl = u'The great wall of China (\u9577\u57ce)'
        self.article_text = u"""This is some text with greek characters\n
                               \u03bc\u1fc6\u03bd\u03b9\u03bd \u1f04\u03b5\u03b9\u03b4\u03b5,
                               \u03b8\u03b5\u03ac,
                               \u03a0\u03b7\u03bb\u03b7\u03ca\u03ac\u03b4\u03b5\u03c9
                               \u1f08\u03c7\u03b9\u03bb\u1fc6\u03bf\u03c2"""
        self.article = amcattest.create_test_article(headline=self.article_hl, project=self.project,
                                                     text=self.article_text, date="2012-01-01")
        self.article = Article.objects.get(pk=self.article.id) # to get date redeserialized
        
        self.articleset = amcattest.create_test_set(project=self.project)
        self.articleset.add(self.article)
        
        self.job = amcattest.create_test_job(project=self.project, articleset=self.articleset,
                                             unitschema=self.schema, articleschema=self.schema)
        
        self.coding = amcattest.create_test_coding(codingjob=self.job, comments="Obvious", article=self.article)
        self.coding.update_values({self.strfield:"abc", self.intfield:1, self.codefield:self.code_d})
示例#23
0
    def test_values_table(self):
        """Does getting a table of values work?"""
        
        schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields()
        job = amcattest.create_test_job(unitschema=schema, articleschema=schema)
        
        c = amcattest.create_test_coding(codingjob=job)
        c.update_values({strf:"bla", intf:1})
	
        self.assertEqual(set(job.values_table().to_list()), {('bla', 1, None)})

        code = amcattest.create_test_code(label="CODED")
        codebook.add_code(code)
        c2 = amcattest.create_test_coding(codingjob=job)
        c2.update_values({intf:-1, codef: code})
        t = job.values_table()
        self.assertEqual(set(t.rows), {c, c2})
        self.assertEqual(set(t.to_list()), {('bla', 1, None), (None, -1, code.id)})
示例#24
0
    def test_schemafield_valid(self):
        schema_with_fields = amcattest.create_test_schema_with_fields()
        schema = schema_with_fields[0]

        self.assertTrue(schemarules_valid(schema))
        self.condition(schema, "()").save()
        self.assertTrue(schemarules_valid(schema))
        self.condition(schema, "(3==2)").save()
        self.assertFalse(schemarules_valid(schema))

        CodingRule.objects.all().delete()

        # Test multiple (correct) rules
        self.condition(schema, "()").save()
        self.condition(schema, "()").save()
        self.condition(schema, "()").save()
        self.assertTrue(schemarules_valid(schema))
        self.condition(schema, "(3==2)").save()
        self.assertFalse(schemarules_valid(schema))
    def test_schemafield_valid(self):
        schema_with_fields = amcattest.create_test_schema_with_fields()
        schema = schema_with_fields[0]

        self.assertTrue(schemarules_valid(schema))
        self.condition(schema, "()").save()
        self.assertTrue(schemarules_valid(schema))
        self.condition(schema, "(3==2)").save()
        self.assertFalse(schemarules_valid(schema))

        CodingRule.objects.all().delete()

        # Test multiple (correct) rules
        self.condition(schema, "()").save()
        self.condition(schema, "()").save()
        self.condition(schema, "()").save()
        self.assertTrue(schemarules_valid(schema))
        self.condition(schema, "(3==2)").save()
        self.assertFalse(schemarules_valid(schema))
示例#26
0
    def test_update_values(self):
        codebook, codes = amcattest.create_test_codebook_with_codes()
        schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
        job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
        articles = list(job.articleset.articles.all())

        coding = amcattest.create_test_coding(codingjob=job, article=articles[0])
        self.assertEqual(0, coding.values.count())
        coding.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
        self.assertEqual(3, coding.values.count())
        self.assertTrue(strf in dict(coding.get_values()))
        self.assertTrue(intf in dict(coding.get_values()))
        self.assertTrue(codef in dict(coding.get_values()))
        self.assertEqual(1, dict(coding.get_values())[intf])

        # Does update_values delete values not present in dict?
        coding.update_values({strf: "blas"})
        self.assertEqual(1, coding.values.count())
        self.assertTrue(strf in dict(coding.get_values()))
        self.assertEqual("blas", dict(coding.get_values())[strf])
示例#27
0
    def test_update_values(self):
        codebook, codes = amcattest.create_test_codebook_with_codes()
        schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
        job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
        articles = list(job.articleset.articles.all())

        coding = amcattest.create_test_coding(codingjob=job, article=articles[0])
        self.assertEqual(0, coding.values.count())
        coding.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
        self.assertEqual(3, coding.values.count())
        self.assertTrue(strf in dict(coding.get_values()))
        self.assertTrue(intf in dict(coding.get_values()))
        self.assertTrue(codef in dict(coding.get_values()))
        self.assertEqual(1, dict(coding.get_values())[intf])

        # Does update_values delete values not present in dict?
        coding.update_values({strf: "blas"})
        self.assertEqual(1, coding.values.count())
        self.assertTrue(strf in dict(coding.get_values()))
        self.assertEqual("blas", dict(coding.get_values())[strf])
示例#28
0
    def test_parse(self):
        import functools

        o1, o2 = amcattest.create_test_code(), amcattest.create_test_code()
        schema_with_fields = amcattest.create_test_schema_with_fields()

        schema = schema_with_fields[0]
        codebook = schema_with_fields[1]

        text_field = schema_with_fields[2]
        number_field = schema_with_fields[3]
        code_field = schema_with_fields[4]

        c = functools.partial(self.condition, schema)

        # Empty conditions should return None
        self.assertEquals(parse(c("")), None)
        self.assertEquals(parse(c("()")), None)

        # Recursion should be checked for
        cr = CodingRule.objects.create(codingschema=schema,
                                       label="foo",
                                       condition="()")
        cr.condition = str(cr.id)
        self.assertRaises(SyntaxError, parse, cr)

        # Nonexisting fields should raise an error
        cr.condition = "0==2"
        self.assertRaises(CodingSchemaField.DoesNotExist, parse, cr)
        cr.condition = "{}==0".format(code_field.id)
        self.assertRaises(Code.DoesNotExist, parse, cr)
        cr.condition = "0"
        self.assertRaises(CodingRule.DoesNotExist, parse, cr)
        cr.condition = "{}=={}".format(code_field.id, o1.id)
        self.assertTrue(parse(cr) is not None)

        # Wrong inputs for fields should raise an error
        for inp in ("'a'", "0.2", "u'a'"):
            cr.condition = "{}=={}".format(number_field.id, inp)
            self.assertRaises(SyntaxError, parse, cr)

        for inp in ("'a'", "0.2", "u'a'", repr(str(o1.id))):
            cr.condition = "{}=={}".format(code_field.id, inp)
            self.assertRaises(SyntaxError, parse, cr)

        for inp in ("'a'", "0.2", "2"):
            cr.condition = "{}=={}".format(text_field.id, inp)
            self.assertRaises(SyntaxError, parse, cr)

        # "Good" inputs shoudl not yield an error
        for field, inp in ((number_field, 1), (text_field, "u'a'"),
                           (code_field, o1.id)):
            cr.condition = "{}=={}".format(field.id, inp)
            self.assertTrue(parse(cr) is not None)

        # Should accept Python-syntax (comments, etc)
        cr.condition = """{}==(
        # This should be a comment)
        {})""".format(text_field.id, "u'a'")
        self.assertTrue(parse(cr) is not None)

        ## Testing output
        tree = parse(c("not {}".format(cr.id)))
        self.assertEquals(tree["type"], NOT)
        self.assertTrue(not isinstance(tree["value"], CodingRule))

        tree = parse(c("{}=={}".format(text_field.id, "u'a'")))
        self.assertEquals(tree, {"type": EQUALS, "values": (text_field, u'a')})

        cr.save()
        tree = parse(c("{cr.id} or {cr.id}".format(cr=cr)))
        self.assertEquals(tree, {"type": OR, "values": (parse(cr), parse(cr))})

        # Should accept greater than / greater or equal to / ...
        parse(c("{number_field.id} > 5".format(**locals())))
        parse(c("{number_field.id} < 5".format(**locals())))
        parse(c("{number_field.id} >= 5".format(**locals())))
        parse(c("{number_field.id} <= 5".format(**locals())))

        # ..but not if schemafieldtype is text or code
        self.assertRaises(SyntaxError, parse,
                          c("{text_field.id} > 5".format(**locals())))
        self.assertRaises(SyntaxError, parse,
                          c("{code_field.id} > 5".format(**locals())))
示例#29
0
    def setUp(self):
        self.s1 = amcattest.create_test_set(5)
        self.a = list(self.s1.articles.all().order_by('id'))
        self.ids = [a.id for a in self.a]

        self.m1 = "Telegraaf"
        self.m2 = "NRC"
        self.m3 = "AD"
        self.a[0].set_property("medium", self.m1)
        self.a[1].set_property("medium", self.m2)
        self.a[2].set_property("medium", self.m2)
        self.a[3].set_property("medium", self.m3)
        self.a[4].set_property("medium", self.m3)

        self.a[0].text = "aap."
        self.a[1].text = "aap. noot."
        self.a[2].text = "aap. noot. mies."

        self.a[0].date = datetime.datetime(2015, 1, 1)
        self.a[1].date = datetime.datetime(2015, 1, 1)
        self.a[2].date = datetime.datetime(2015, 2, 1)
        self.a[3].date = datetime.datetime(2016, 1, 1)
        self.a[4].date = datetime.datetime(2016, 1, 1)
        self.a[0].save()
        self.a[1].save()
        self.a[2].save()
        self.a[3].save()
        self.a[4].save()
        
        # Uncomment if ever using elastic :)
        # self.s1.refresh_index(full_refresh=True)

        self.schema, self.codebook, self.strf, self.intf, self.codef, self.boolf, self.qualf = (
            amcattest.create_test_schema_with_fields(isarticleschema=True))

        self.sschema, self.scodebook, self.sstrf, self.sintf, self.scodef, self.sboolf, self.squalf = (
            amcattest.create_test_schema_with_fields(isarticleschema=False))

        # Article
        self.codes = self.codebook.get_codes()
        self.code_A, = [c for c in self.codes if c.label == "A"]
        self.code_B, = [c for c in self.codes if c.label == "B"]
        self.code_A1, = [c for c in self.codes if c.label == "A1"]

        # Sentence
        self.scodes = self.codebook.get_codes()
        self.scode_A, = [c for c in self.scodes if c.label == "A"]
        self.scode_B, = [c for c in self.scodes if c.label == "B"]
        self.scode_A1, = [c for c in self.scodes if c.label == "A1"]
        self.scode_A1b, = [c for c in self.scodes if c.label == "A1b"]

        # Does not get fired in unit test?
        for article in [self.a[0], self.a[1], self.a[2], self.a[3], self.a[4]]:
            get_or_create_sentences(article)

        self.job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema, unitschema=self.sschema)

        self.c1 = amcattest.create_test_coding(codingjob=self.job, article=self.a[0])
        self.c1.update_values({self.codef: self.code_A.id, self.intf: 4, self.qualf: 4})

        self.c2 = amcattest.create_test_coding(codingjob=self.job, article=self.a[1])
        self.c2.update_values({self.codef: self.code_A.id, self.intf: 2, self.qualf: 1})

        self.c3 = amcattest.create_test_coding(codingjob=self.job, article=self.a[2])
        self.c3.update_values({self.codef: self.code_B.id, self.intf: 1, self.qualf: 2})

        self.c4 = amcattest.create_test_coding(codingjob=self.job, article=self.a[3])
        self.c4.update_values({self.codef: self.code_A1.id, self.intf: 1})

        self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a[0], sentence=self.a[0].sentences.all()[0])
        self.sentence_coding.update_values({self.scodef: self.scode_A1.id, self.sintf: 1})

        self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a[2], sentence=self.a[2].sentences.all()[0])
        self.sentence_coding.update_values({self.scodef: self.scode_A1.id, self.sintf: 1})

        self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a[2], sentence=self.a[2].sentences.all()[0])
        self.sentence_coding.update_values({self.scodef: self.scode_A1b.id, self.sintf: 1})

        # Try to confuse aggregator by inserting multiple codingjobs
        job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema)
        c4 = amcattest.create_test_coding(codingjob=job, article=self.a[2])
        c4.update_values({self.codef: self.code_B.id, self.intf: 10, self.qualf: 8})
示例#30
0
    def setUp(self):
        self.s1 = amcattest.create_test_set(5)
        self.a1, self.a2, self.a3, self.a4, self.a5 = self.s1.articles.all()
        self.m1 = "Telegraaf"
        self.m2 = "NRC"
        self.m3 = "AD"
        self.a1.set_property("medium", self.m1)
        self.a2.set_property("medium", self.m2)
        self.a3.set_property("medium", self.m2)
        self.a4.set_property("medium", self.m3)
        self.a5.set_property("medium", self.m3)

        self.a1.text = "aap."
        self.a2.text = "aap. noot."
        self.a3.text = "aap. noot. mies."

        self.a1.date = datetime.datetime(2015, 1, 1)
        self.a2.date = datetime.datetime(2015, 1, 1)
        self.a3.date = datetime.datetime(2015, 2, 1)
        self.a4.date = datetime.datetime(2016, 1, 1)
        self.a5.date = datetime.datetime(2016, 1, 1)
        self.a1.save()
        self.a2.save()
        self.a3.save()
        self.a4.save()
        self.a5.save()

        # Uncomment if ever using elastic :)
        # self.s1.refresh_index(full_refresh=True)

        self.schema, self.codebook, self.strf, self.intf, self.codef, self.boolf, self.qualf = (
            amcattest.create_test_schema_with_fields(isarticleschema=True))

        self.sschema, self.scodebook, self.sstrf, self.sintf, self.scodef, self.sboolf, self.squalf = (
            amcattest.create_test_schema_with_fields(isarticleschema=False))

        # Article
        self.codes = self.codebook.get_codes()
        self.code_A, = [c for c in self.codes if c.label == "A"]
        self.code_B, = [c for c in self.codes if c.label == "B"]
        self.code_A1, = [c for c in self.codes if c.label == "A1"]

        # Sentence
        self.scodes = self.codebook.get_codes()
        self.scode_A, = [c for c in self.scodes if c.label == "A"]
        self.scode_B, = [c for c in self.scodes if c.label == "B"]
        self.scode_A1, = [c for c in self.scodes if c.label == "A1"]

        # Does not get fired in unit test?
        for article in [self.a1, self.a2, self.a3, self.a4, self.a5]:
            get_or_create_sentences(article)

        self.job = amcattest.create_test_job(articleset=self.s1,
                                             articleschema=self.schema,
                                             unitschema=self.sschema)

        self.c1 = amcattest.create_test_coding(codingjob=self.job,
                                               article=self.a1)
        self.c1.update_values({
            self.codef: self.code_A.id,
            self.intf: 4,
            self.qualf: 4
        })

        self.c2 = amcattest.create_test_coding(codingjob=self.job,
                                               article=self.a2)
        self.c2.update_values({
            self.codef: self.code_A.id,
            self.intf: 2,
            self.qualf: 1
        })

        self.c3 = amcattest.create_test_coding(codingjob=self.job,
                                               article=self.a3)
        self.c3.update_values({
            self.codef: self.code_B.id,
            self.intf: 1,
            self.qualf: 2
        })

        self.c4 = amcattest.create_test_coding(codingjob=self.job,
                                               article=self.a4)
        self.c4.update_values({self.codef: self.code_A1.id, self.intf: 1})

        self.sentence_coding = amcattest.create_test_coding(
            codingjob=self.job,
            article=self.a1,
            sentence=self.a1.sentences.all()[0])
        self.sentence_coding.update_values({
            self.scodef: self.scode_A1.id,
            self.sintf: 1
        })

        # Try to confuse aggregator by inserting multiple codingjobs
        job = amcattest.create_test_job(articleset=self.s1,
                                        articleschema=self.schema)
        c4 = amcattest.create_test_coding(codingjob=job, article=self.a3)
        c4.update_values({
            self.codef: self.code_B.id,
            self.intf: 10,
            self.qualf: 8
        })
示例#31
0
    def test_replace_codings(self):
        schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(
            isarticleschema=True)
        schema2, codebook2, strf2, intf2, codef2, _, _ = amcattest.create_test_schema_with_fields(
            isarticleschema=True)
        codingjob = amcattest.create_test_job(articleschema=schema,
                                              narticles=10)

        coded_article = CodedArticle.objects.get(
            article=codingjob.articleset.articles.all()[0],
            codingjob=codingjob)
        coded_article.replace_codings(
            [self._get_coding_dict(intval=10, field_id=codef.id)])

        self.assertEqual(1, coded_article.codings.all().count())
        self.assertEqual(1,
                         coded_article.codings.all()[0].values.all().count())
        coding = coded_article.codings.all()[0]
        value = coding.values.all()[0]
        self.assertEqual(coding.sentence, None)
        self.assertEqual(value.strval, None)
        self.assertEqual(value.intval, 10)
        self.assertEqual(value.field, codef)

        # Overwrite previous coding
        coded_article.replace_codings(
            [self._get_coding_dict(intval=11, field_id=intf.id)])
        self.assertEqual(1, coded_article.codings.all().count())
        self.assertEqual(1,
                         coded_article.codings.all()[0].values.all().count())
        coding = coded_article.codings.all()[0]
        value = coding.values.all()[0]
        self.assertEqual(coding.sentence, None)
        self.assertEqual(value.strval, None)
        self.assertEqual(value.intval, 11)
        self.assertEqual(value.field, intf)

        # Try to insert illigal values
        illval1 = self._get_coding_dict(intval=1, strval="a", field_id=intf.id)
        illval2 = self._get_coding_dict(field_id=intf.id)
        illval3 = self._get_coding_dict(intval=1)
        illval4 = self._get_coding_dict(intval=1, field_id=strf2.id)

        self.assertRaises(ValueError, coded_article.replace_codings, [illval1])
        self.assertRaises(ValueError, coded_article.replace_codings, [illval2])
        self.assertRaises(IntegrityError, coded_article.replace_codings,
                          [illval3])
        self.assertRaises(ValueError, coded_article.replace_codings, [illval4])

        # Unspecified values default to None
        val = self._get_coding_dict(intval=1, field_id=intf.id)
        del val["values"][0]["strval"]
        coded_article.replace_codings([val])
        value = coded_article.codings.all()[0].values.all()[0]
        self.assertEqual(value.strval, None)
        self.assertEqual(value.intval, 1)

        val = self._get_coding_dict(strval="a", field_id=intf.id)
        del val["values"][0]["intval"]
        coded_article.replace_codings([val])
        value = coded_article.codings.all()[0].values.all()[0]
        self.assertEqual(value.strval, "a")
        self.assertEqual(value.intval, None)
    def test_parse(self):
        import functools

        o1, o2 = amcattest.create_test_code(), amcattest.create_test_code()
        schema_with_fields = amcattest.create_test_schema_with_fields()

        schema = schema_with_fields[0]
        codebook = schema_with_fields[1]

        text_field = schema_with_fields[2]
        number_field = schema_with_fields[3]
        code_field = schema_with_fields[4]

        c = functools.partial(self.condition, schema)
        
        # Empty conditions should return None
        self.assertEquals(parse(c("")), None)
        self.assertEquals(parse(c("()")), None)
        
        # Recursion should be checked for
        cr = CodingRule.objects.create(codingschema=schema, label="foo", condition="()")
        cr.condition = str(cr.id)
        self.assertRaises(SyntaxError, parse, cr)

        # Nonexisting fields should raise an error
        cr.condition = "0==2"
        self.assertRaises(CodingSchemaField.DoesNotExist, parse, cr)
        cr.condition = "{}==0".format(code_field.id)
        self.assertRaises(Code.DoesNotExist, parse, cr)
        cr.condition = "0"
        self.assertRaises(CodingRule.DoesNotExist, parse, cr)
        cr.condition = "{}=={}".format(code_field.id, o1.id)
        self.assertTrue(parse(cr) is not None)

        # Wrong inputs for fields should raise an error
        for inp in ("'a'", "0.2", "u'a'"):
            cr.condition = "{}=={}".format(number_field.id, inp)
            self.assertRaises(SyntaxError, parse, cr)

        for inp in ("'a'", "0.2", "u'a'", repr(str(o1.id))):
            cr.condition = "{}=={}".format(code_field.id, inp)
            self.assertRaises(SyntaxError, parse, cr)

        for inp in ("'a'", "0.2", "2"):
            cr.condition = "{}=={}".format(text_field.id, inp)
            self.assertRaises(SyntaxError, parse, cr)
        
        # "Good" inputs shoudl not yield an error
        for field, inp in ((number_field, 1), (text_field, "u'a'"), (code_field, o1.id)):
            cr.condition = "{}=={}".format(field.id, inp)
            self.assertTrue(parse(cr) is not None)

        # Should accept Python-syntax (comments, etc)
        cr.condition = """{}==(
        # This should be a comment)
        {})""".format(text_field.id, "u'a'")
        self.assertTrue(parse(cr) is not None)

        ## Testing output
        tree = parse(c("not {}".format(cr.id)))
        self.assertEquals(tree["type"], NOT)
        self.assertTrue(not isinstance(tree["value"], CodingRule))

        tree = parse(c("{}=={}".format(text_field.id, "u'a'")))
        self.assertEquals(tree, {"type":EQUALS, "values":(text_field, u'a')})

        cr.save()
        tree = parse(c("{cr.id} or {cr.id}".format(cr=cr)))
        self.assertEquals(tree, {"type":OR, "values":(parse(cr), parse(cr))})

        # Should accept greater than / greater or equal to / ...
        parse(c("{number_field.id} > 5".format(**locals())))
        parse(c("{number_field.id} < 5".format(**locals())))
        parse(c("{number_field.id} >= 5".format(**locals())))
        parse(c("{number_field.id} <= 5".format(**locals())))

        # ..but not if schemafieldtype is text or code
        self.assertRaises(SyntaxError, parse, c("{text_field.id} > 5".format(**locals())))
        self.assertRaises(SyntaxError, parse, c("{code_field.id} > 5".format(**locals())))
示例#33
0
    def setUp(self):
        """Populate a project with articles, coding etc"""
        from amcat.models import Language, Article

        self.project = amcattest.create_test_project()

        # create a codebook
        self.codebook = amcattest.create_test_codebook(project=self.project,
                                                       name="base codebook")
        en = Language.objects.get(label='en')
        sv = Language.objects.create(label='sv')
        _cargs = dict(language=en, codebook=self.codebook)
        self.code_a = amcattest.create_test_code(label="a", **_cargs)
        self.code_a.add_label(label=u"\xe5", language=sv)
        self.code_b = amcattest.create_test_code(label="b",
                                                 parent=self.code_a,
                                                 **_cargs)
        self.code_c = amcattest.create_test_code(label="c",
                                                 parent=self.code_a,
                                                 **_cargs)

        # Codebook bases not yet implemented
        self.sub_codebook = amcattest.create_test_codebook(
            project=self.project, name="sub codebook")
        #self.sub_codebook.add_base(self.codebook)
        self.code_d = amcattest.create_test_code(label="d",
                                                 language=en,
                                                 codebook=self.sub_codebook,
                                                 parent=self.code_a)
        CodebookCode.objects.create(codebook=self.sub_codebook,
                                    code=self.code_c,
                                    hide=True)
        CodebookCode.objects.create(codebook=self.sub_codebook,
                                    code=self.code_b,
                                    parent=None)

        # create a schema
        self.schema, _dummy, self.strfield, self.intfield, self.codefield = (
            amcattest.create_test_schema_with_fields(
                project=self.project, codebook=self.sub_codebook))

        self.article_hl = u'The great wall of China (\u9577\u57ce)'
        self.article_text = u"""This is some text with greek characters\n
                               \u03bc\u1fc6\u03bd\u03b9\u03bd \u1f04\u03b5\u03b9\u03b4\u03b5,
                               \u03b8\u03b5\u03ac,
                               \u03a0\u03b7\u03bb\u03b7\u03ca\u03ac\u03b4\u03b5\u03c9
                               \u1f08\u03c7\u03b9\u03bb\u1fc6\u03bf\u03c2"""
        self.article = amcattest.create_test_article(headline=self.article_hl,
                                                     project=self.project,
                                                     text=self.article_text,
                                                     date="2012-01-01")
        self.article = Article.objects.get(
            pk=self.article.id)  # to get date redeserialized

        self.articleset = amcattest.create_test_set(project=self.project)
        self.articleset.add(self.article)

        self.job = amcattest.create_test_job(project=self.project,
                                             articleset=self.articleset,
                                             unitschema=self.schema,
                                             articleschema=self.schema)

        self.coding = amcattest.create_test_coding(codingjob=self.job,
                                                   comments="Obvious",
                                                   article=self.article)
        self.coding.update_values({
            self.strfield: "abc",
            self.intfield: 1,
            self.codefield: self.code_d
        })
示例#34
0
    def setUp(self):
        self.s1 = amcattest.create_test_set(5)
        self.a1, self.a2, self.a3, self.a4, self.a5 = self.s1.articles.all()
        self.m1 = self.a1.medium
        self.m2 = self.a2.medium
        self.m3 = self.a3.medium
        self.m4 = self.a4.medium
        self.a3.medium = self.m2
        self.a3.save()

        self.a1.text = "aap."
        self.a2.text = "aap. noot."
        self.a3.text = "aap. noot. mies."

        self.a1.date = datetime.datetime(2015, 0o1, 0o1)
        self.a2.date = datetime.datetime(2015, 0o1, 0o1)
        self.a3.date = datetime.datetime(2015, 0o2, 0o1)
        self.a4.date = datetime.datetime(2016, 0o1, 0o1)
        self.a5.date = datetime.datetime(2016, 0o1, 0o1)
        self.a1.save()
        self.a2.save()
        self.a3.save()
        self.a4.save()
        self.a5.save()

        # Uncomment if ever using elastic :)
        # self.s1.refresh_index(full_refresh=True)

        self.schema, self.codebook, self.strf, self.intf, self.codef, self.boolf, self.qualf = (
            amcattest.create_test_schema_with_fields(isarticleschema=True))

        self.sschema, self.scodebook, self.sstrf, self.sintf, self.scodef, self.sboolf, self.squalf = (
            amcattest.create_test_schema_with_fields(isarticleschema=False))

        # Article
        self.codes = self.codebook.get_codes()
        self.code_A, = [c for c in self.codes if c.label == "A"]
        self.code_B, = [c for c in self.codes if c.label == "B"]
        self.code_A1, = [c for c in self.codes if c.label == "A1"]

        # Sentence
        self.scodes = self.codebook.get_codes()
        self.scode_A, = [c for c in self.scodes if c.label == "A"]
        self.scode_B, = [c for c in self.scodes if c.label == "B"]
        self.scode_A1, = [c for c in self.scodes if c.label == "A1"]

        # Does not get fired in unit test?
        for article in [self.a1, self.a2, self.a3, self.a4, self.a5]:
            get_or_create_sentences(article)

        self.job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema, unitschema=self.sschema)

        self.c1 = amcattest.create_test_coding(codingjob=self.job, article=self.a1)
        self.c1.update_values({self.codef: self.code_A.id, self.intf: 4, self.qualf: 4})

        self.c2 = amcattest.create_test_coding(codingjob=self.job, article=self.a2)
        self.c2.update_values({self.codef: self.code_A.id, self.intf: 2, self.qualf: 1})

        self.c3 = amcattest.create_test_coding(codingjob=self.job, article=self.a3)
        self.c3.update_values({self.codef: self.code_B.id, self.intf: 1, self.qualf: 2})

        self.c4 = amcattest.create_test_coding(codingjob=self.job, article=self.a4)
        self.c4.update_values({self.codef: self.code_A1.id, self.intf: 1})

        self.sentence_coding = amcattest.create_test_coding(codingjob=self.job, article=self.a1, sentence=self.a1.sentences.all()[0])
        self.sentence_coding.update_values({self.scodef: self.scode_A1.id, self.sintf: 1})

        # Try to confuse aggregator by inserting multiple codingjobs
        job = amcattest.create_test_job(articleset=self.s1, articleschema=self.schema)
        c4 = amcattest.create_test_coding(codingjob=job, article=self.a3)
        c4.update_values({self.codef: self.code_B.id, self.intf: 10, self.qualf: 8})