def test_codebook_to_xml(self):
# Empty codebook
cb = codebook_to_xml(amcattest.create_test_codebook())
self.assertEquals(cb.tag, "codebook")
children = [c.tag for c in cb.iterchildren()]
for prop in ("project_id", "name", "roots", "id"):
self.assertTrue(prop in children)
self.assertFalse(len([c for c in cb.find("roots")]))
# Codebook with two roots
cb = codebook_to_xml(amcattest.create_test_codebook_with_codes()[0])
self.assertEquals([c.find("label").text for c in cb.find("roots")], ["A", "B"])
self.assertTrue("A1a" in etree.tostring(cb))
# Test unicode
cb = amcattest.create_test_codebook_with_codes()[0]
c = cb.codes[0]
c.label = u"\u2603" # It's Mr. Snowman!
c.save()
# Shouldn't raise errors
cb = codebook_to_xml(Codebook.objects.get(id=cb.id))
etree.tostring(cb)
def test_codebook_to_xml(self):
# Empty codebook
cb = codebook_to_xml(amcattest.create_test_codebook())
self.assertEquals(cb.tag, "codebook")
children = [c.tag for c in cb.iterchildren()]
for prop in ("project_id", "name", "roots", "id"):
self.assertTrue(prop in children)
self.assertFalse(len([c for c in cb.find("roots")]))
# Codebook with two roots
cb = codebook_to_xml(amcattest.create_test_codebook_with_codes()[0])
self.assertEquals([c.find("label").text for c in cb.find("roots")],
["A", "B"])
self.assertTrue("A1a" in etree.tostring(cb))
# Test unicode
cb = amcattest.create_test_codebook_with_codes()[0]
label = cb.codes[0].labels.all()[0]
label.label = u"\u2603" # It's Mr. Snowman!
label.save()
# Shouldn't raise errors
cb = codebook_to_xml(Codebook.objects.get(id=cb.id))
etree.tostring(cb)
def test_nqueries(self):
from amcat.tools import amcatlogging
amcatlogging.setup()
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
log.info(codes)
amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
c = list(codingjobs[0].codings)[0]
amcatlogging.debug_module('django.db.backends')
script = self._get_results_script([job], {strf : {}, intf : {}})
with self.checkMaxQueries(8):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(ids=True)})
with self.checkMaxQueries(8):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(labels=True)})
with self.checkMaxQueries(8):
list(csv.reader(StringIO(script.run())))
def test_results(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook,
isarticleschema=True)
sschema, codebook, sstrf, sintf, scodef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=sschema, articleschema=schema, narticles=5)
articles = list(job.articleset.articles.all())
c = amcattest.create_test_coding(codingjob=job, article=articles[0])
# test simple coding with a codebook code
c.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
self.assertEqual(self._get_results([job], {strf: {}, intf: {}, codef: dict(ids=True)}),
[('bla', 1, codes["A1b"].id)])
# test multiple codings and parents
c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
c2.update_values({strf: "blx", intf: 1, codef: codes["B1"].id})
self.assertEqual(set(self._get_results([job], {strf: {}, intf: {}, codef: dict(labels=True, parents=2)})),
{('bla', 1, "A", "A1", "A1b"), ('blx', 1, "B", "B1", "B1")})
# test sentence result
s = amcattest.create_test_sentence(article=articles[0])
sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
sc.update_values({sstrf: "z", sintf: -1, scodef: codes["A"].id})
self.assertEqual(set(self._get_results([job], {strf: {}, sstrf: {}, sintf: {}}, export_level=2)),
{('bla', 'z', -1), ('blx', None, None)})
def test_non_unique_label(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
codes["A1"].label = "A"
codes["A1"].save()
self.assertRaises(DuplicateLabelError, MediumCategory, codebook=codebook)
def test_nqueries(self):
from amcat.tools import amcatlogging
amcatlogging.setup()
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values({strf:"bla", intf:1, codef:codes["A1b"]})
codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
c = codingjobs[0].codings.all()[0]
amcatlogging.debug_module('django.db.backends')
script = self._get_results_script([job], {strf : {}, intf : {}})
with self.checkMaxQueries(5):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(ids=True)})
with self.checkMaxQueries(5):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf : {}, intf : {}, codef : dict(labels=True)})
with self.checkMaxQueries(5):
list(csv.reader(StringIO(script.run())))
def test_nqueries(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
log.info(codes)
amcattest.create_test_coding(codingjob=job, article=articles[0]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[1]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[2]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[3]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
amcattest.create_test_coding(codingjob=job, article=articles[4]).update_values(
{strf: "bla", intf: 1, codef: codes["A1b"].id})
codingjobs = list(CodingJob.objects.filter(pk__in=[job.id]))
c = list(codingjobs[0].codings)[0]
script = self._get_results_script([job], {strf: {}, intf: {}})
with self.checkMaxQueries(9):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf: {}, intf: {}, codef: dict(ids=True)})
with self.checkMaxQueries(9):
list(csv.reader(StringIO(script.run())))
script = self._get_results_script([job], {strf: {}, intf: {}, codef: dict(labels=True)})
with self.checkMaxQueries(9):
list(csv.reader(StringIO(script.run())))
def test_results(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook, isarticleschema=True)
sschema, codebook, sstrf, sintf, scodef = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=sschema, articleschema=schema, narticles=5)
articles = list(job.articleset.articles.all())
c = amcattest.create_test_coding(codingjob=job, article=articles[0])
# test simple coding with a codebook code
c.update_values({strf:"bla", intf:1, codef:codes["A1b"].id})
self.assertEqual(self._get_results([job], {strf : {}, intf : {}, codef : dict(ids=True)}),
[('bla', 1, codes["A1b"].id)])
# test multiple codings and parents
c2 = amcattest.create_test_coding(codingjob=job, article=articles[1])
c2.update_values({strf:"blx", intf:1, codef:codes["B1"].id})
self.assertEqual(set(self._get_results([job], {strf : {}, intf : {}, codef : dict(labels=True, parents=2)})),
{('bla', 1, "A", "A1", "A1b"), ('blx', 1, "B", "B1", "B1")})
# test sentence result
s = amcattest.create_test_sentence(article=articles[0])
sc = amcattest.create_test_coding(codingjob=job, article=articles[0], sentence=s)
sc.update_values({sstrf:"z", sintf:-1, scodef:codes["A"].id})
self.assertEqual(set(self._get_results([job], {strf : {}, sstrf : {}, sintf : {}}, export_level=2)),
{('bla', 'z', -1), ('blx', None, None)})
def setUp(self):
self.de = Language.objects.get(label="de")
self.nl = Language.objects.get(label="nl")
self.default = Language.objects.get(id=1)
self.codebook = amcattest.create_test_codebook_with_codes()[0]
self.codes_list = sorted(self.codebook.codes.all(), key=lambda c:c.id)
self.codes_list[0].add_label(self.de, "Ein")
self.codes_list[1].add_label(self.nl, "Een")
def setUp(self):
self.de = Language.objects.get(label="de")
self.nl = Language.objects.get(label="nl")
self.default = Language.objects.get(id=1)
self.codebook = amcattest.create_test_codebook_with_codes()[0]
self.codes_list = sorted(self.codebook.codes.all(), key=lambda c: c.id)
self.codes_list[0].add_label(self.de, "Ein")
self.codes_list[1].add_label(self.nl, "Een")
def get_form(self, **kwargs):
codebook = None
if "project" in kwargs:
project = kwargs.pop("project")
else:
codebook = amcattest.create_test_codebook_with_codes()[0]
project = codebook.project
return project, codebook, SelectionForm(project, data=kwargs)
def test_invalid_reference(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
Medium.objects.bulk_create(Medium(name=l) for l in codes)
# Should raise no error..
MediumCategory(codebook=codebook)
# Delete one medium, so one reference doesn't exist
Medium.objects.all()[0].delete()
self.assertRaises(InvalidReferenceError, MediumCategory, codebook=codebook)
def get_form(self, **kwargs):
codebook = None
if "project" in kwargs:
project = kwargs.pop("project")
else:
codebook = amcattest.create_test_codebook_with_codes()[0]
project = codebook.project
return project, codebook, SelectionForm(project, data=kwargs)
def test_clean_tree(self):
import functools
o1, o2 = amcattest.create_test_code(), amcattest.create_test_code()
codebook, codebook_codes = amcattest.create_test_codebook_with_codes()
schema_with_fields = amcattest.create_test_schema_with_fields(codebook=codebook)
schema = schema_with_fields[0]
code_field = schema_with_fields[4]
c = functools.partial(self.condition, schema)
tree = parse(c("{code_field.id}=={o1.id}".format(**locals())))
self.assertRaises(ValidationError, clean_tree, schema, tree)
tree = parse(c("{code_field.id}=={code.id}".format(code_field=code_field, code=codebook.codes[0])))
self.assertEquals(clean_tree(schema, tree), None)
self.assertRaises(ValidationError, clean_tree, amcattest.create_test_schema_with_fields()[0], tree)
def test_create_missing(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
Medium.objects.bulk_create(Medium(name=l) for l in codes)
# Should raise no error..
MediumCategory(codebook=codebook)
# Delete one medium, so one reference doesn't exist
medium = Medium.objects.all()[0]
medium.delete()
self.assertRaises(Medium.DoesNotExist, Medium.objects.get, id=medium.id)
self.assertRaises(Medium.DoesNotExist, Medium.objects.get, name=medium.name)
# Medium should exist again after instantiating MediumCategory
MediumCategory(codebook=codebook, create_missing=True)
self.assertRaises(Medium.DoesNotExist, Medium.objects.get, id=medium.id)
self.assertEqual(1, Medium.objects.filter(name=medium.name).count())
def test_update_values(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef, _, _ = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
coding = amcattest.create_test_coding(codingjob=job, article=articles[0])
self.assertEqual(0, coding.values.count())
coding.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
self.assertEqual(3, coding.values.count())
self.assertTrue(strf in dict(coding.get_values()))
self.assertTrue(intf in dict(coding.get_values()))
self.assertTrue(codef in dict(coding.get_values()))
self.assertEqual(1, dict(coding.get_values())[intf])
# Does update_values delete values not present in dict?
coding.update_values({strf: "blas"})
self.assertEqual(1, coding.values.count())
self.assertTrue(strf in dict(coding.get_values()))
self.assertEqual("blas", dict(coding.get_values())[strf])
def test_update_values(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
schema, codebook, strf, intf, codef = amcattest.create_test_schema_with_fields(codebook=codebook)
job = amcattest.create_test_job(unitschema=schema, articleschema=schema, narticles=7)
articles = list(job.articleset.articles.all())
coding = amcattest.create_test_coding(codingjob=job, article=articles[0])
self.assertEqual(0, coding.values.count())
coding.update_values({strf: "bla", intf: 1, codef: codes["A1b"].id})
self.assertEqual(3, coding.values.count())
self.assertTrue(strf in dict(coding.get_values()))
self.assertTrue(intf in dict(coding.get_values()))
self.assertTrue(codef in dict(coding.get_values()))
self.assertEqual(1, dict(coding.get_values())[intf])
# Does update_values delete values not present in dict?
coding.update_values({strf: "blas"})
self.assertEqual(1, coding.values.count())
self.assertTrue(strf in dict(coding.get_values()))
self.assertEqual("blas", dict(coding.get_values())[strf])
def test_clean_tree(self):
import functools
o1, o2 = amcattest.create_test_code(), amcattest.create_test_code()
codebook, codebook_codes = amcattest.create_test_codebook_with_codes()
schema_with_fields = amcattest.create_test_schema_with_fields(
codebook=codebook)
schema = schema_with_fields[0]
code_field = schema_with_fields[4]
c = functools.partial(self.condition, schema)
tree = parse(c("{code_field.id}=={o1.id}".format(**locals())))
self.assertRaises(ValidationError, clean_tree, schema, tree)
tree = parse(
c("{code_field.id}=={code.id}".format(code_field=code_field,
code=codebook.codes[0])))
self.assertEquals(clean_tree(schema, tree), None)
self.assertRaises(ValidationError, clean_tree,
amcattest.create_test_schema_with_fields()[0], tree)
def test_aggregate(self):
codebook, codes = amcattest.create_test_codebook_with_codes()
Medium.objects.bulk_create(Medium(name=l) for l in codes)
category = MediumCategory(codebook=codebook)
value = AverageValue(CodingSchemaField())
# Should all collapse into 'A'
medium_A = Medium.objects.get(name="A")
medium_A1 = Medium.objects.get(name="A1")
medium_A1b = Medium.objects.get(name="A1b")
rows = [
# Medium count average
[medium_A.id, 1, Decimal(1) ],
[medium_A1.id, 2, Decimal(2) ],
[medium_A1b.id, 1, Decimal(3)]
]
self.assertEqual(
set(map(tuple, category.aggregate([category], value, rows))),
{(medium_A.id, 4, Decimal(2))}
)
