def test_validation(self):
"""Test whether codebookcode validation works"""
a, b, c, d = [amcattest.create_test_code(label=l) for l in "abcd"]
A = amcattest.create_test_codebook(name="A")
self.assertTrue(A.add_code(a, b, hide=True) is not None)
# Delete code added in previous statement
A.delete_codebookcode(CodebookCode.objects.get(codebook=A, code=a, parent=b))
A.delete_codebookcode(CodebookCode.objects.get(codebook=A, code=b))
self.assertRaises(ValueError, A.add_code, a, validfrom=datetime.datetime(2010, 1, 1),
validto=datetime.datetime(1900, 1, 1))
A.add_code(a)
A.add_code(b)
A.add_code(c, a)
self.assertRaises(ValueError, A.add_code, c, a)
self.assertRaises(ValueError, A.add_code, c, b)
self.assertRaises(ValueError, A.add_code, c, hide=True)
A.add_code(d, a, validto=datetime.datetime(2010, 1, 1))
A.add_code(d, b, validfrom=datetime.datetime(2010, 1, 1))
self.assertRaises(ValueError, A.add_code, d, a)
self.assertRaises(ValueError, A.add_code, d, a, validto=datetime.datetime(1900, 1, 1))
self.assertRaises(ValueError, A.add_code, d, a, validfrom=datetime.datetime(1900, 1, 1))
# different code book should be ok
B = amcattest.create_test_codebook(name="B")
B.add_code(a)
B.add_code(c, a)
def test_validation(self):
"""Test whether codebookcode validation works"""
a, b, c, d = [amcattest.create_test_code(label=l) for l in "abcd"]
A = amcattest.create_test_codebook(name="A")
self.assertTrue(A.add_code(a, b, hide=True) is not None)
# Delete code added in previous statement
A.delete_codebookcode(CodebookCode.objects.get(codebook=A, code=a, parent=b))
A.delete_codebookcode(CodebookCode.objects.get(codebook=A, code=b))
self.assertRaises(ValueError, A.add_code, a, validfrom=datetime.datetime(2010, 1, 1),
validto=datetime.datetime(1900, 1, 1))
A.add_code(a)
A.add_code(b)
A.add_code(c, a)
self.assertRaises(ValueError, A.add_code, c, a)
self.assertRaises(ValueError, A.add_code, c, b)
self.assertRaises(ValueError, A.add_code, c, hide=True)
A.add_code(d, a, validto=datetime.datetime(2010, 1, 1))
A.add_code(d, b, validfrom=datetime.datetime(2010, 1, 1))
self.assertRaises(ValueError, A.add_code, d, a)
self.assertRaises(ValueError, A.add_code, d, a, validto=datetime.datetime(1900, 1, 1))
self.assertRaises(ValueError, A.add_code, d, a, validfrom=datetime.datetime(1900, 1, 1))
# different code book should be ok
B = amcattest.create_test_codebook(name="B")
B.add_code(a)
B.add_code(c, a)
def test_codebook_avg(self):
aggr = self._get_aggr(flat=True)
# D: a
# +b
# +a1
D = amcattest.create_test_codebook(name="D")
D.add_code(self.code_A)
D.add_code(self.code_B, self.code_A)
D.add_code(self.code_A1, self.code_A)
cbsf = GroupedCodebookFieldCategory(self.codef, codebook=D, level=1)
result = set(aggr.get_aggregate([cbsf], [AverageValue(self.intf)]))
result = {(c, av) for (c, (av, _)) in result}
self.assertEqual(result, {(self.code_A, 2.0)})
# E: a
# +b
# a1
E = amcattest.create_test_codebook(name="E")
E.add_code(self.code_A)
E.add_code(self.code_B, self.code_A)
E.add_code(self.code_A1)
cbsf = GroupedCodebookFieldCategory(self.codef, codebook=E, level=1)
result = set(aggr.get_aggregate([cbsf], [AverageValue(self.intf)]))
result = {(c, av) for (c, (av, _)) in result}
self.assertEqual(result, {
(self.code_A1, 1.0),
(self.code_A, 7.0/3.0),
})
def test_hierarchy(self):
"""Does the code/parent base class resolution work"""
a, b, c, d, e, f = [amcattest.create_test_code(label=l) for l in "abcdef"]
# A: b (validto = 2010)
# +a
#
# a should have parent b, even when requesting hierarchy of 2013.
Z = amcattest.create_test_codebook(name="Z")
Z.add_code(code=b, validto=datetime.datetime(2010, 1, 1))
Z.add_code(code=a, parent=b)
tree = Z.get_tree(datetime.datetime(2013, 1, 1))
self.assertEqual(tree[0].children[0].label, 'a')
# A: a
# +b
# +c
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b, a)
A.add_code(c, b)
self.assertEqual(self.standardize(A), 'a:None;b:a;c:b')
# D: d
# +e
# +f
D = amcattest.create_test_codebook(name="D")
D.add_code(d)
D.add_code(e, d)
D.add_code(f, d)
self.assertEqual(self.standardize(D), 'd:None;e:d;f:d')
def test_codebook_avg(self):
aggr = self._get_aggr(flat=True)
# D: a
# +b
# +a1
D = amcattest.create_test_codebook(name="D")
D.add_code(self.code_A)
D.add_code(self.code_B, self.code_A)
D.add_code(self.code_A1, self.code_A)
cbsf = SchemafieldCategory(self.codef, codebook=D)
result = set(aggr.get_aggregate([cbsf], [AverageValue(self.intf)]))
self.assertEqual(result, {(self.code_A, 2.0)})
# E: a
# +b
# a1
E = amcattest.create_test_codebook(name="E")
E.add_code(self.code_A)
E.add_code(self.code_B, self.code_A)
E.add_code(self.code_A1)
cbsf = SchemafieldCategory(self.codef, codebook=E)
result = set(aggr.get_aggregate([cbsf], [AverageValue(self.intf)]))
self.assertEqual(result, {
(self.code_A1, 1.0),
(self.code_A, 7.0 / 3.0),
})
def test_codebookcodes(self):
"""Test the get_codebookcodes function"""
def _copairs(codebook, code):
"""Get (child, parent) pairs for codebookcodes"""
for co in codebook.get_codebookcodes(code):
yield co.code, co.parent
a, b, c, d, e = [amcattest.create_test_code(label=l) for l in "abcde"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b)
A.add_code(c,
a,
validfrom=datetime(1910, 1, 1),
validto=datetime(1920, 1, 1))
A.add_code(c,
b,
validfrom=datetime(1920, 1, 1),
validto=datetime(1930, 1, 1))
A.add_code(d, a)
A.add_code(e, a)
self.assertEqual(set(_copairs(A, a)), set([(a, None)]))
self.assertEqual(set(_copairs(A, c)), set([(c, a), (c, b)]))
self.assertEqual(set(_copairs(A, d)), set([(d, a)]))
self.assertEqual(set(_copairs(A, e)), set([(e, a)]))
B = amcattest.create_test_codebook(name="B")
B.add_code(d, b)
B.add_code(e, b, validfrom=datetime(2012, 1, 1))
def test_hierarchy(self):
"""Does the code/parent base class resolution work"""
import datetime
a, b, c, d, e, f = [
amcattest.create_test_code(label=l) for l in "abcdef"
]
# A: b (validto = 2010)
# +a
#
# a should have parent b, even when requesting hierarchy of 2013.
Z = amcattest.create_test_codebook(name="Z")
Z.add_code(code=b, validto=datetime.datetime(2010, 1, 1))
Z.add_code(code=a, parent=b)
tree = Z.get_tree(datetime.datetime(2013, 1, 1))
self.assertEqual(tree[0].children[0].label, 'a')
# A: a
# +b
# +c
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b, a)
A.add_code(c, b)
self.assertEqual(self.standardize(A), 'a:None;b:a;c:b')
# D: d
# +e
# +f
D = amcattest.create_test_codebook(name="D")
D.add_code(d)
D.add_code(e, d)
D.add_code(f, d)
self.assertEqual(self.standardize(D), 'd:None;e:d;f:d')
def test_codebookcodes(self):
"""Test the get_codebookcodes function"""
def _copairs(codebook, code):
"""Get (child, parent) pairs for codebookcodes"""
for co in codebook.get_codebookcodes(code):
yield co.code, co.parent
a, b, c, d, e = [amcattest.create_test_code(label=l) for l in "abcde"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b)
A.add_code(c, a, validfrom=datetime.datetime(1910, 1, 1), validto=datetime.datetime(1920, 1, 1))
A.add_code(c, b, validfrom=datetime.datetime(1920, 1, 1), validto=datetime.datetime(1930, 1, 1))
A.add_code(d, a)
A.add_code(e, a)
self.assertEqual(set(_copairs(A, a)), {(a, None)})
self.assertEqual(set(_copairs(A, c)), {(c, a), (c, b)})
self.assertEqual(set(_copairs(A, d)), {(d, a)})
self.assertEqual(set(_copairs(A, e)), {(e, a)})
B = amcattest.create_test_codebook(name="B")
B.add_code(d, b)
B.add_code(e, b, validfrom=datetime.datetime(2012, 1, 1))
def test_codebook_avg(self):
aggr = self._get_aggr(flat=True)
# D: a
# +b
# +a1
D = amcattest.create_test_codebook(name="D")
D.add_code(self.code_A)
D.add_code(self.code_B, self.code_A)
D.add_code(self.code_A1, self.code_A)
cbsf = SchemafieldCategory(self.codef, codebook=D)
result = set(aggr.get_aggregate([cbsf], [AverageValue(self.intf)]))
self.assertEqual(result, {(self.code_A, 2.0)})
# E: a
# +b
# a1
E = amcattest.create_test_codebook(name="E")
E.add_code(self.code_A)
E.add_code(self.code_B, self.code_A)
E.add_code(self.code_A1)
cbsf = SchemafieldCategory(self.codef, codebook=E)
result = set(aggr.get_aggregate([cbsf], [AverageValue(self.intf)]))
self.assertEqual(result, {
(self.code_A1, 1.0),
(self.code_A, 7.0/3.0),
})
def test_get_codes(self):
"""Does get_codes work without using too many queries?"""
codes = [amcattest.create_test_code(label=l) for l in "abcdef"]
hiddencodes = [amcattest.create_test_code(label=l) for l in "abcdef"]
B = amcattest.create_test_codebook(name="A")
map(B.add_code, codes)
B.add_code(hiddencodes[0])
A = amcattest.create_test_codebook(name="A")
[A.add_code(c, hide=True) for c in hiddencodes]
A.add_base(B)
clear_cache(Code)
with self.checkMaxQueries(2, "Bases for codebooks"):
list(A.bases)
list(B.bases)
with self.checkMaxQueries(2, "Get Code ids"):
self.assertEqual(set(A.get_code_ids()), set(c.id for c in codes))
clear_cache(Code)
with self.checkMaxQueries(2, "Get Codes"):
self.assertEqual(set(A.get_codes()), set(codes))
clear_cache(Code)
with self.checkMaxQueries(2, "Hidden Codes"):
self.assertEqual(set(A.get_codes(include_hidden=True)),
set(codes) | set(hiddencodes))
with self.checkMaxQueries(0, "Cached codes per codebook"):
self.assertEqual(set(A.get_codes(include_hidden=True)),
set(codes) | set(hiddencodes))
def test_codebookserialiser_memoisation(self):
"""Does memoisation work?"""
A = amcattest.create_test_codebook(name="A")
B = amcattest.create_test_codebook(name="B")
s1 = CodebookSerialiser(_DummyField(A))
s2 = CodebookSerialiser(_DummyField(A))
self.assertIs(s1, s2)
s3 = CodebookSerialiser(_DummyField(B))
self.assertNotEqual(s2, s3)
def test_codebookserialiser_memoisation(self):
"""Does memoisation work?"""
A = amcattest.create_test_codebook(name="A")
B = amcattest.create_test_codebook(name="B")
s1 = CodebookSerialiser(_DummyField(A))
s2 = CodebookSerialiser(_DummyField(A))
self.assertIs(s1, s2)
s3 = CodebookSerialiser(_DummyField(B))
self.assertNotEqual(s2, s3)
def test_get_codebook(self):
"""Test whether using get_codebook results in shared objects"""
cid = amcattest.create_test_codebook().pk
c1 = get_codebook(cid)
c2 = get_codebook(cid)
self.assertIs(c1, c2)
c3 = amcattest.create_test_codebook()
self.assertIsNot(c1, c3)
c4 = get_codebook(c3.id)
self.assertEqual(c3, c4)
self.assertIsNot(c1, c4)
self.assertNotEqual(c1, c4)
def test_cache_labels(self):
"""Does caching labels work?"""
from amcat.models.language import Language
lang = Language.objects.get(pk=1)
codes = set(
amcattest.create_test_code(label=l, language=lang)
for l in "abcdef")
morecodes = set(
amcattest.create_test_code(label=l, language=lang)
for l in "abcdef")
h = amcattest.create_test_code(label="hidden", language=lang)
B = amcattest.create_test_codebook(name="B")
map(B.add_code, morecodes)
A = amcattest.create_test_codebook(name="A")
map(A.add_code, codes)
A.add_code(h, hide=True)
C = amcattest.create_test_codebook(name="C")
A.add_base(C)
C.add_base(B)
n_bases = 3
clear_cache(Code)
maxq = n_bases * 2 + 1 # base + codebookcodes per base, codes
with self.checkMaxQueries(maxq, "Cache Codes"):
self.assertEqual(set(A.get_codes(include_hidden=True)),
codes | set([h]) | morecodes)
with self.checkMaxQueries(1, "Cache labels"): # labels
A.cache_labels(lang)
with self.checkMaxQueries(0, "Cache labels again"):
A.cache_labels(lang)
A._cache_labels_languages = set()
clear_cache(Code)
with self.checkMaxQueries(2, "Cache labels directly"): # codes, labels
A.cache_labels(lang)
with self.checkMaxQueries(0, "Print labels"):
for c in codes:
c.get_label(lang)
str(c)
unicode(c)
repr(c)
with self.checkMaxQueries(0, "Print labels via hierarchy"):
for c, _p in A.get_hierarchy(include_hidden=True):
c.get_label(lang)
str(c)
unicode(c)
def test_get_ancestors(self):
a, b, c, d, e, f = [amcattest.create_test_code(label=l) for l in "abcdef"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b)
A.add_code(c, b)
A.add_code(e, a)
A.add_code(d, c)
A.add_code(f, a)
self.assertEqual(list(A.get_ancestor_ids(f.id)), [f.id, a.id])
self.assertEqual(list(A.get_ancestor_ids(a.id)), [a.id])
B = amcattest.create_test_codebook(name="B")
B.add_code(f, b)
def test_get_ancestors(self):
a, b, c, d, e, f = [amcattest.create_test_code(label=l) for l in "abcdef"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b)
A.add_code(c, b)
A.add_code(e, a)
A.add_code(d, c)
A.add_code(f, a)
self.assertEqual(list(A.get_ancestor_ids(f.id)), [f.id, a.id])
self.assertEqual(list(A.get_ancestor_ids(a.id)), [a.id])
B = amcattest.create_test_codebook(name="B")
B.add_code(f, b)
def test_codes(self):
"""Test the codes property"""
a, b, c, d = [amcattest.create_test_code(label=l) for l in "abcd"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b, a)
A.add_code(c, a, validfrom=datetime.datetime(1910, 1, 1), validto=datetime.datetime(1920, 1, 1))
self.assertEqual(set(A.codes), {a, b, c})
B = amcattest.create_test_codebook(name="B")
B.add_code(d, b)
self.assertEqual(set(B.codes), {d, b})
def test_codes(self):
"""Test the codes property"""
a, b, c, d = [amcattest.create_test_code(label=l) for l in "abcd"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b, a)
A.add_code(c, a, validfrom=datetime.datetime(1910, 1, 1), validto=datetime.datetime(1920, 1, 1))
self.assertEqual(set(A.codes), {a, b, c})
B = amcattest.create_test_codebook(name="B")
B.add_code(d, b)
self.assertEqual(set(B.codes), {d, b})
def test_ordering(self):
"""
Codebookcodes should always be returned in order, according
to their codenr-property (see CodebookCode.codenr).
"""
cb = amcattest.create_test_codebook()
code_a, code_b, code_c = (amcattest.create_test_code() for i in range(3))
ccode_c = cb.add_code(code_c, ordernr=3)
ccode_b = cb.add_code(code_b, ordernr=2)
ccode_a = cb.add_code(code_a, ordernr=1)
cb.invalidate_cache()
# These tests will automatically be run with caching enabled by
# the method test_caching_correctness()
self.assertEquals(tuple(cb.codebookcodes), (ccode_a, ccode_b, ccode_c))
roots = [ti.code_id for ti in cb.get_tree()]
self.assertEquals(roots, [code_a.id, code_b.id, code_c.id])
# Test again with differnt order
ccode_b.ordernr = 256
ccode_b.save()
cb.invalidate_cache()
self.assertEquals(tuple(cb.codebookcodes), (ccode_a, ccode_c, ccode_b))
roots = [ti.code_id for ti in cb.get_tree()]
self.assertEquals(roots, [code_a.id, code_c.id, code_b.id])
def test_codebook_to_xml(self):
# Empty codebook
cb = codebook_to_xml(amcattest.create_test_codebook())
self.assertEquals(cb.tag, "codebook")
children = [c.tag for c in cb.iterchildren()]
for prop in ("project_id", "name", "roots", "id"):
self.assertTrue(prop in children)
self.assertFalse(len([c for c in cb.find("roots")]))
# Codebook with two roots
cb = codebook_to_xml(amcattest.create_test_codebook_with_codes()[0])
self.assertEquals([c.find("label").text for c in cb.find("roots")],
["A", "B"])
self.assertTrue("A1a" in etree.tostring(cb))
# Test unicode
cb = amcattest.create_test_codebook_with_codes()[0]
label = cb.codes[0].labels.all()[0]
label.label = u"\u2603" # It's Mr. Snowman!
label.save()
# Shouldn't raise errors
cb = codebook_to_xml(Codebook.objects.get(id=cb.id))
etree.tostring(cb)
def test_codebookserialiser(self):
"""Test the codebook serialiser"""
from amcat.models.language import Language
A = amcattest.create_test_codebook(name="A")
c = amcattest.create_test_code(label="bla")
l2 = Language.objects.create(label="XXX")
c.add_label(language=l2, label="blx")
A.add_code(c)
s = CodebookSerialiser(_DummyField(A))
self.assertEqual(s.serialise(c), c.id)
self.assertEqual(s.deserialise(c.id), c)
d = amcattest.create_test_code(label="not in codebook")
#(de)serialising a code from outside codebook should not raise errors:
self.assertEqual(s.serialise(d), d.id)
self.assertEqual(s.deserialise(d.id), d)
self.assertRaises(Exception, s.deserialise, -9999999999999999)
self.assertEqual(set([c]), set(s.possible_values))
self.assertEqual("bla", s.value_label(c))
self.assertEqual("blx", s.value_label(c, l2))
def test_cache_labels_language(self):
"""Does caching labels for multiple language work
esp. caching non-existence of a label"""
from amcat.models.language import Language
l1 = Language.objects.get(pk=1)
a = amcattest.create_test_code(label="a", language=l1)
l2 = Language.objects.get(pk=2)
b = amcattest.create_test_code(label="b", language=l2)
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b)
with self.checkMaxQueries(4, "Cache labels"):
A.cache_labels(l1)
A.cache_labels(l2)
a, b = map(get_code, [a.id, b.id])
with self.checkMaxQueries(0, "Get exisitng labels"):
self.assertEqual(a.get_label(l1), "a")
self.assertEqual(b.get_label(l2), "b")
with self.checkMaxQueries(0, "Get non-existing labels"):
self.assertEqual(a.get_label(l2, l1), "a")
self.assertEqual(b.get_label(l1, l2), "b")
def test_get_aggregation_mapping(self):
a, b, c, d, e, f,g = [amcattest.create_test_code(label=l) for l in "abcdefg"]
# D: d
# +e
# +f
# ++g
# a
# b
# +c
D = amcattest.create_test_codebook(name="D")
D.add_code(d)
D.add_code(e, d)
D.add_code(f, d)
D.add_code(g, f)
D.add_code(a)
D.add_code(b)
D.add_code(c, b)
# Codebook not cached
self.assertRaises(ValueError, D.get_aggregation_mapping)
D.cache()
# Codebook cached
self.assertEqual({u'c': u'b', u'e': u'd', u'f': u'd'}, D.get_aggregation_mapping())
def test_codebookserialiser(self):
"""Test the codebook serialiser"""
from amcat.models.language import Language
A = amcattest.create_test_codebook(name="A")
c = amcattest.create_test_code(label="bla")
l2 = Language.objects.create(label="XXX")
c.add_label(language=l2, label="blx")
A.add_code(c)
s = CodebookSerialiser(_DummyField(A))
self.assertEqual(s.serialise(c), c.id)
self.assertEqual(s.deserialise(c.id), c)
d = amcattest.create_test_code(label="not in codebook")
#(de)serialising a code from outside codebook should not raise errors:
self.assertEqual(s.serialise(d), d.id)
self.assertEqual(s.deserialise(d.id), d)
self.assertRaises(Exception, s.deserialise, -9999999999999999)
self.assertEqual(set([c]), set(s.possible_values))
self.assertEqual("bla", s.value_label(c))
self.assertEqual("blx", s.value_label(c, l2))
def setUp(self):
from amcat.models.coding.coding import CodingValue
# create a coding job set with a sensible schema and some articles to 'code'
self.schema = amcattest.create_test_schema()
self.codebook = amcattest.create_test_codebook()
self.code = amcattest.create_test_code(label="CODED")
self.codebook.add_code(self.code)
texttype = CodingSchemaFieldType.objects.get(pk=1)
inttype = CodingSchemaFieldType.objects.get(pk=2)
codetype = CodingSchemaFieldType.objects.get(pk=5)
create = CodingSchemaField.objects.create
self.textfield = create(codingschema=self.schema,
fieldnr=1,
fieldtype=texttype,
label="Text")
self.intfield = create(codingschema=self.schema,
fieldnr=2,
fieldtype=inttype,
label="Number")
self.codefield = create(codingschema=self.schema,
fieldnr=3,
fieldtype=codetype,
label="Code",
codebook=self.codebook)
self.users = [amcattest.create_test_user() for _x in range(2)]
self.articles, self.jobs, self.asets = [], [], []
for i, user in enumerate([0, 0, 0, 0, 1]):
aset = amcattest.create_test_set(articles=2 * (i + 1))
self.articles += list(aset.articles.all())
self.asets.append(aset)
job = amcattest.create_test_job(articleschema=self.schema,
unitschema=self.schema,
coder=self.users[user],
articleset=aset)
self.jobs.append(job)
self.an1 = Coding.objects.create(codingjob=self.jobs[0],
article=self.articles[0])
self.an2 = Coding.objects.create(codingjob=self.jobs[0],
article=self.articles[1])
self.an2.set_status(STATUS_COMPLETE)
self.an2.comments = 'Makkie!'
self.an2.save()
sent = amcattest.create_test_sentence()
self.sa1 = Coding.objects.create(codingjob=self.jobs[0],
article=self.articles[0],
sentence=sent)
self.sa2 = Coding.objects.create(codingjob=self.jobs[0],
article=self.articles[0],
sentence=sent)
create = CodingValue.objects.create
create(coding=self.sa1, field=self.intfield, intval=1)
create(coding=self.sa1, field=self.textfield, strval="bla")
create(coding=self.sa2, field=self.textfield, strval="blx")
create(coding=self.sa1, field=self.codefield, intval=self.code.id)
def test_codebook_to_xml(self):
# Empty codebook
cb = codebook_to_xml(amcattest.create_test_codebook())
self.assertEquals(cb.tag, "codebook")
children = [c.tag for c in cb.iterchildren()]
for prop in ("project_id", "name", "roots", "id"):
self.assertTrue(prop in children)
self.assertFalse(len([c for c in cb.find("roots")]))
# Codebook with two roots
cb = codebook_to_xml(amcattest.create_test_codebook_with_codes()[0])
self.assertEquals([c.find("label").text for c in cb.find("roots")], ["A", "B"])
self.assertTrue("A1a" in etree.tostring(cb))
# Test unicode
cb = amcattest.create_test_codebook_with_codes()[0]
c = cb.codes[0]
c.label = u"\u2603" # It's Mr. Snowman!
c.save()
# Shouldn't raise errors
cb = codebook_to_xml(Codebook.objects.get(id=cb.id))
etree.tostring(cb)
def test_get_descendants(self):
a, b, c, d, e, f,g = [amcattest.create_test_code(label=l) for l in "abcdefg"]
# D: d
# +e
# +f
# ++g
# a
# b
# +c
D = amcattest.create_test_codebook(name="D")
D.add_code(d)
D.add_code(e, d)
D.add_code(f, d)
D.add_code(g, f)
D.add_code(a)
D.add_code(b)
D.add_code(c, b)
tree = D.get_tree()
a = next(t for t in tree if t.label == "a")
b = next(t for t in tree if t.label == "b")
d = next(t for t in tree if t.label == "d")
self.assertEqual({t.code_id for t in a.get_descendants()}, set())
self.assertEqual({t.code_id for t in b.get_descendants()}, {c.id})
self.assertEqual({t.code_id for t in d.get_descendants()}, {e.id, f.id, g.id})
def test_codebook_views(self):
codebook = amcattest.create_test_codebook(project=self.project)
self._test_get_post_write_access(reverse("navigator:codebook-import", args=[self.project.id]))
self._test_get_post_write_access(reverse("navigator:codebook-link", args=[self.project.id]))
self._test_get_post_write_access(reverse("navigator:codebook-delete", args=[self.project.id, codebook.id]))
self._test_get_post_write_access(reverse("navigator:codebook-change-name", args=[self.project.id, codebook.id]))
def test_lexicon(self):
from amcat.models import Language
cb = amcattest.create_test_codebook()
l = Language.objects.create(label="lexlang")
c1 = amcattest.create_test_code(label="a")
c1.add_label(l, "A")
cb.add_code(c1)
c2 = amcattest.create_test_code(label="b")
c2.add_label(l, "B1, B2")
cb.add_code(c2)
r = RuleSet.objects.create(label="test",
lexicon_codebook=cb,
lexicon_language=l)
result = sorted(r.get_lexicon(), key=lambda l: l['lexclass'])
self.assertEqual(result, [{
"lexclass": "a",
"lemma": ["A"]
}, {
"lexclass": "b",
"lemma": ["B1", "B2"]
}])
def test_get_aggregation_mapping(self):
a, b, c, d, e, f,g = [amcattest.create_test_code(label=l) for l in "abcdefg"]
# D: d
# +e
# +f
# ++g
# a
# b
# +c
D = amcattest.create_test_codebook(name="D")
D.add_code(d)
D.add_code(e, d)
D.add_code(f, d)
D.add_code(g, f)
D.add_code(a)
D.add_code(b)
D.add_code(c, b)
# Codebook not cached
self.assertRaises(ValueError, D.get_aggregation_mapping)
D.cache()
# Codebook cached
self.assertEqual({u'c': u'b', u'e': u'd', u'f': u'd'}, D.get_aggregation_mapping())
def test_ordering(self):
"""
Codebookcodes should always be returned in order, according
to their codenr-property (see CodebookCode.codenr).
"""
cb = amcattest.create_test_codebook()
code_a, code_b, code_c = (amcattest.create_test_code()
for i in range(3))
ccode_c = cb.add_code(code_c, ordernr=3)
ccode_b = cb.add_code(code_b, ordernr=2)
ccode_a = cb.add_code(code_a, ordernr=1)
cb.invalidate_cache()
# These tests will automatically be run with caching enabled by
# the method test_caching_correctness()
self.assertEquals(tuple(cb.codebookcodes), (ccode_a, ccode_b, ccode_c))
roots = [ti.code_id for ti in cb.get_tree()]
self.assertEquals(roots, [code_a.id, code_b.id, code_c.id])
# Test again with differnt order
ccode_b.ordernr = 256
ccode_b.save()
cb.invalidate_cache()
self.assertEquals(tuple(cb.codebookcodes), (ccode_a, ccode_c, ccode_b))
roots = [ti.code_id for ti in cb.get_tree()]
self.assertEquals(roots, [code_a.id, code_c.id, code_b.id])
def test_codebook_base_lookup(self):
"""Test whether looking things up in a code book with bases is efficient"""
A = amcattest.create_test_codebook()
for i in range(100):
c = A.add_code(amcattest.create_test_code(label=str(i)), None)
B = amcattest.create_test_codebook(name="D+B")
B.add_base(A)
for i in range(100):
B.add_code(amcattest.create_test_code(label=str(i)), None)
# 2 to get bases, 2 to get codes, 1 to get base codebook
with self.checkMaxQueries(5, "Subcodebook code list"):
self.assertEqual(len(list(B.codes)), 200)
with self.checkMaxQueries(0, "Subcodebook code membership (cached)"):
self.assertIn(c.code, B.codes)
def setUp(self):
super(TestNQueries, self).setUp()
self.texttype = CodingSchemaFieldType.objects.get(pk=1)
self.inttype = CodingSchemaFieldType.objects.get(pk=2)
self.codetype = CodingSchemaFieldType.objects.get(pk=5)
self.codebook = amcattest.create_test_codebook()
self.code = self.codebook.add_code(
amcattest.create_test_code(label="CODED")).code
def _get_test_codebook(self, lexicon_language, codes):
"""@param codes: a dict of label : lexical entry"""
cb = amcattest.create_test_codebook()
for label, lexical in codes.iteritems():
c = amcattest.create_test_code(codebook=cb, label=label)
c.add_label(lexicon_language, lexical)
cb.cache_labels(1)
return cb
def setUp(self):
if not User.objects.filter(username='******').exists():
create_test_user(username='******', password='******')
self.cb = amcattest.create_test_codebook()
from django.test import Client
self.client = Client()
response = self.client.post('/accounts/login/', {'username': '******', 'password': '******'})
self.assert_status(response, 302)
def setUp(self):
"""Populate a project with articles, coding etc"""
from amcat.models import Language, Article
self.project = amcattest.create_test_project()
# create a codebook
self.codebook = amcattest.create_test_codebook(project=self.project, name="base codebook")
en = Language.objects.get(label='en')
sv = Language.objects.create(label='sv')
_cargs = dict(language=en, codebook=self.codebook)
self.code_a = amcattest.create_test_code(label="a", **_cargs)
self.code_a.add_label(label=u"\xe5", language=sv)
self.code_b = amcattest.create_test_code(label="b", parent=self.code_a, **_cargs)
self.code_c = amcattest.create_test_code(label="c", parent=self.code_a, **_cargs)
# Codebook bases not yet implemented
self.sub_codebook = amcattest.create_test_codebook(project=self.project, name="sub codebook")
#self.sub_codebook.add_base(self.codebook)
self.code_d = amcattest.create_test_code(label="d", language=en,
codebook=self.sub_codebook, parent=self.code_a)
CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_c, hide=True)
CodebookCode.objects.create(codebook=self.sub_codebook, code=self.code_b, parent=None)
# create a schema
self.schema, _dummy, self.strfield, self.intfield, self.codefield = (
amcattest.create_test_schema_with_fields(project=self.project, codebook=self.sub_codebook))
self.article_hl = u'The great wall of China (\u9577\u57ce)'
self.article_text = u"""This is some text with greek characters\n
\u03bc\u1fc6\u03bd\u03b9\u03bd \u1f04\u03b5\u03b9\u03b4\u03b5,
\u03b8\u03b5\u03ac,
\u03a0\u03b7\u03bb\u03b7\u03ca\u03ac\u03b4\u03b5\u03c9
\u1f08\u03c7\u03b9\u03bb\u1fc6\u03bf\u03c2"""
self.article = amcattest.create_test_article(headline=self.article_hl, project=self.project,
text=self.article_text, date="2012-01-01")
self.article = Article.objects.get(pk=self.article.id) # to get date redeserialized
self.articleset = amcattest.create_test_set(project=self.project)
self.articleset.add(self.article)
self.job = amcattest.create_test_job(project=self.project, articleset=self.articleset,
unitschema=self.schema, articleschema=self.schema)
self.coding = amcattest.create_test_coding(codingjob=self.job, comments="Obvious", article=self.article)
self.coding.update_values({self.strfield:"abc", self.intfield:1, self.codefield:self.code_d})
def _get_test_codebook(self, lexicon_language, codes):
"""@param codes: a dict of label : lexical entry"""
cb = amcattest.create_test_codebook()
for label, lexical in codes.iteritems():
c = amcattest.create_test_code(codebook=cb, label=label)
c.add_label(lexicon_language, lexical)
cb.cache_labels(1)
return cb
def _get_test_codebook(self, codes, l_lang, r_lang):
codebook = create_test_codebook()
code_dict = {}
for l_label, r_label, parent_label in codes:
parent = code_dict[parent_label] if parent_label else None
code = codebook.create_code(label=l_label, language=l_lang, parent=parent)
code.code.add_label(language=r_lang, label=r_label)
code_dict[l_label] = code
return codebook
def setUp(self):
username = "******".format(uuid4())
user = create_test_user(username=username, password='******')
self.project = amcattest.create_test_project(owner=user)
self.cb = amcattest.create_test_codebook(project=self.project)
from django.test import Client
self.client = Client()
success = self.client.login(username=username, password='******')
self.assertTrue(success)
def test_circular(self):
"""Are circular bases prevented from saving?"""
A, B, C, D, E = [amcattest.create_test_codebook() for _x in range(5)]
A.add_base(B)
self.assertRaises(ValidationError, B.add_base, A)
A.add_base(C)
A.add_base(D)
A.add_base(E)
D.add_base(B)
self.assertRaises(ValidationError, B.add_base, A)
def test_codebook_lookup(self):
"""Test whether looking things up in a code book is efficient"""
A = amcattest.create_test_codebook()
last = None
for i in range(100):
last = A.add_code(amcattest.create_test_code(label=str(i)), last)
with self.checkMaxQueries(2, "Code membership"):
_x = [a.id for a in A.codes]
with self.checkMaxQueries(0, "Code membership (cached)"):
_x = [a.id for a in A.codes]
def test_create(self):
"""Can we create objects?"""
c = amcattest.create_test_codebook()
o = amcattest.create_test_code()
co = c.add_code(o)
co2 = c.add_code(Code.objects.create(), parent=o)
self.assertIn(co, c.codebookcodes)
#self.assertIn(o, c.codes)
self.assertEqual(co2.parent, o)
def _get_test_codebook(self, codes, l_lang, r_lang):
codebook = create_test_codebook()
code_dict = {}
for l_label, r_label, parent_label in codes:
parent = code_dict[parent_label] if parent_label else None
code = codebook.create_code(label=l_label,
language=l_lang,
parent=parent)
code.code.add_label(language=r_lang, label=r_label)
code_dict[l_label] = code
return codebook
def test_create(self):
"""Can we create objects?"""
c = amcattest.create_test_codebook()
o = amcattest.create_test_code()
co = c.add_code(o)
co2 = c.add_code(Code.objects.create(), parent=o)
self.assertIn(co, c.codebookcodes)
#self.assertIn(o, c.codes)
self.assertEqual(co2.parent, o)
def test_codes(self):
"""Test the codes property"""
a, b, c, d = [amcattest.create_test_code(label=l) for l in "abcd"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b, a)
A.add_code(c,
a,
validfrom=datetime(1910, 1, 1),
validto=datetime(1920, 1, 1))
self.assertEqual(A.codes, set([a, b, c]))
B = amcattest.create_test_codebook(name="B")
B.add_code(d, b)
self.assertEqual(B.codes, set([d]))
B.add_base(A)
with self.checkMaxQueries(2, "Getting codes from base"):
self.assertEqual(B.codes, set([a, b, c, d]))
def test_get_timebound_functions(self):
"""Test whether time-bound functions are returned correctly"""
a, b, c = [amcattest.create_test_code(label=l) for l in "abc"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(c, a, validfrom=datetime.datetime(2010, 1, 1))
self.assertEqual(self.standardize(A), 'a:None;c:a')
self.assertEqual(self.standardize(A, date=datetime.datetime(1900, 1, 1)), 'a:None')
A.add_code(b)
A.add_code(c, b, validto=datetime.datetime(2010, 1, 1))
self.assertEqual(self.standardize(A, date=datetime.datetime(2009, 12, 31)), 'a:None;b:None;c:b')
self.assertEqual(self.standardize(A, date=datetime.datetime(2010, 1, 1)), 'a:None;b:None;c:a')
self.assertEqual(self.standardize(A, date=datetime.datetime(2010, 1, 2)), 'a:None;b:None;c:a')
def test_get_timebound_functions(self):
"""Test whether time-bound functions are returned correctly"""
a, b, c = [amcattest.create_test_code(label=l) for l in "abc"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(c, a, validfrom=datetime.datetime(2010, 1, 1))
self.assertEqual(self.standardize(A), 'a:None;c:a')
self.assertEqual(self.standardize(A, date=datetime.datetime(1900, 1, 1)), 'a:None')
A.add_code(b)
A.add_code(c, b, validto=datetime.datetime(2010, 1, 1))
self.assertEqual(self.standardize(A, date=datetime.datetime(2009, 12, 31)), 'a:None;b:None;c:b')
self.assertEqual(self.standardize(A, date=datetime.datetime(2010, 1, 1)), 'a:None;b:None;c:a')
self.assertEqual(self.standardize(A, date=datetime.datetime(2010, 1, 2)), 'a:None;b:None;c:a')
def test_codebook_labels(self):
"""Does caching the codebook_labels work"""
language = Language.objects.get(pk=1)
A = amcattest.create_test_codebook()
for i in range(100):
A.add_code(
amcattest.create_test_code(label=str(i), language=language),
None)
codes = list(A.codes)
with self.checkMaxQueries(1, "Caching labels for a codebook"):
A.cache_labels(language)
with self.checkMaxQueries(0, "Getting cached labels for a codebook"):
for x in codes:
x.get_label(language)
def test_get_language_ids(self):
al, bl, cl = [Language.objects.create(label=l) for l in "abc"]
ac, bc, cc = [amcattest.create_test_code(label=l) for l in "abc"]
ac.add_label(al, "a")
bc.add_label(bl, "a")
cc.add_label(cl, "a")
A = amcattest.create_test_codebook(name="A")
A.add_code(ac)
A.add_code(bc)
A.add_code(cc)
self.assertEqual(A.get_language_ids(), {al.id, bl.id, cl.id})
def test_roots_children(self):
"""Does getting the roots and children work?"""
a, b, c, d, e, f = [amcattest.create_test_code(label=l) for l in "abcdef"]
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b)
A.add_code(e, a)
A.add_code(d, c)
A.add_code(f, a)
self.assertEqual(set(A.get_roots()), {a, b, c})
self.assertEqual(set(A.get_children(a)), {e, f})
self.assertEqual(set(A.get_children(c)), {d})
self.assertEqual(set(A.get_children(d)), set())
def test_get_language_ids(self):
al, bl, cl = [Language.objects.create(label=l) for l in "abc"]
ac, bc, cc = [amcattest.create_test_code(label=l) for l in "abc"]
ac.add_label(al, "a")
bc.add_label(bl, "a")
cc.add_label(cl, "a")
A = amcattest.create_test_codebook(name="A")
A.add_code(ac)
A.add_code(bc)
A.add_code(cc)
self.assertEqual(A.get_language_ids(), {al.id, bl.id, cl.id})
def setUp(self):
from amcat.models.coding.coding import CodingValue
# create a coding job set with a sensible schema and some articles to 'code'
self.schema = amcattest.create_test_schema()
self.codebook = amcattest.create_test_codebook()
self.code = amcattest.create_test_code(label="CODED")
self.codebook.add_code(self.code)
texttype = CodingSchemaFieldType.objects.get(pk=1)
inttype = CodingSchemaFieldType.objects.get(pk=2)
codetype = CodingSchemaFieldType.objects.get(pk=5)
create = CodingSchemaField.objects.create
self.textfield = create(codingschema=self.schema, fieldnr=1, fieldtype=texttype,
label="Text")
self.intfield = create(codingschema=self.schema, fieldnr=2, fieldtype=inttype,
label="Number")
self.codefield = create(codingschema=self.schema, fieldnr=3, fieldtype=codetype,
label="Code", codebook=self.codebook)
self.users = [amcattest.create_test_user() for _x in range(2)]
self.articles, self.jobs, self.asets = [], [], []
for i, user in enumerate([0, 0, 0, 0, 1]):
aset = amcattest.create_test_set(articles=2 * (i+1))
self.articles += list(aset.articles.all())
self.asets.append(aset)
job = amcattest.create_test_job(articleschema=self.schema, unitschema=self.schema,
coder = self.users[user], articleset=aset)
self.jobs.append(job)
self.an1 = Coding.objects.create(codingjob=self.jobs[0], article=self.articles[0])
self.an2 = Coding.objects.create(codingjob=self.jobs[0], article=self.articles[1])
self.an2.set_status(STATUS_COMPLETE)
self.an2.comments = 'Makkie!'
self.an2.save()
sent = amcattest.create_test_sentence()
self.sa1 = Coding.objects.create(codingjob=self.jobs[0], article=self.articles[0],
sentence=sent)
self.sa2 = Coding.objects.create(codingjob=self.jobs[0], article=self.articles[0],
sentence=sent)
create = CodingValue.objects.create
create(coding=self.sa1, field=self.intfield, intval=1)
create(coding=self.sa1, field=self.textfield, strval="bla")
create(coding=self.sa2, field=self.textfield, strval="blx")
create(coding=self.sa1, field=self.codefield, intval=self.code.id)
def test_cache_labels_language(self):
"""Does caching labels for multiple language work
esp. caching non-existence of a label"""
from amcat.models.language import Language
l1 = Language.objects.get(pk=1)
a = amcattest.create_test_code(extra_label="a", extra_language=l1)
l2 = Language.objects.get(pk=2)
b = amcattest.create_test_code(extra_label="b", extra_language=l2)
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
A.add_code(b)
with self.checkMaxQueries(3, "Cache labels"):
# 1 for "normal" caching, and 1 for each cache_labels()
A.cache_labels(l1)
A.cache_labels(l2)
# Codebooks must export _lables, which is a dictionary which
# contains all the cached labels.
self.assertEqual(A._labels[a.id][l1.id], "a")
self.assertEqual(A._labels[b.id][l2.id], "b")
A.invalidate_cache()
with self.checkMaxQueries(2, "Cache labels"):
A.cache_labels(l1, l2)
with self.checkMaxQueries(0, "Get cached codes"):
a, b = map(A.get_code, [a.id, b.id])
with self.checkMaxQueries(0, "Get multiple cached codes"):
list(A.get_codes())
with self.checkMaxQueries(0, "Get exisitng labels"):
self.assertEqual(a.get_label(l1), "a")
self.assertEqual(b.get_label(l2), "b")
with self.checkMaxQueries(0, "Get non-existing labels"):
self.assertRaises(ObjectDoesNotExist, a.get_label, l2)
with self.checkMaxQueries(0, "Get tree"):
list(A.get_tree())
c = amcattest.create_test_code(label="c")
with self.checkMaxQueries(5, "Add new code"):
A.add_code(c)
def test_label(self):
from amcat.models import Language
l = Language.objects.create(id=13, label="query")
a = amcattest.create_test_code(label="test")
A = amcattest.create_test_codebook(name="A")
A.add_code(a)
a = A.get_code(a.id)
t = GetQueries(codebook=A.id).run()
result = set(tuple(row) for row in t)
self.assertEqual(result, set([(a.id, "test", None)]))
a.add_label(l, "bla")
t = GetQueries(codebook=A.id).run()
result = set(tuple(row) for row in t)
self.assertEqual(result, set([(a.id, "test", "bla")]))
def test_lexicon(self):
from amcat.models import Language
cb = amcattest.create_test_codebook()
l = Language.objects.create(label="lexlang")
c1 = amcattest.create_test_code(label="a")
c1.add_label(l, "A")
cb.add_code(c1)
c2 = amcattest.create_test_code(label="b")
c2.add_label(l, "B1, B2")
cb.add_code(c2)
r = RuleSet.objects.create(label="test", lexicon_codebook=cb,
lexicon_language=l)
result = sorted(r.get_lexicon(), key=lambda l: l['lexclass'])
self.assertEqual(result, [{"lexclass": "a", "lemma": ["A"]},
{"lexclass": "b", "lemma": ["B1", "B2"]}])
def test_add_codes(self):
a, b, c, d, e = [amcattest.create_test_code(label=l) for l in "abcde"]
A = amcattest.create_test_codebook(name="A")
A.add_codes([a])
self.assertTrue(A._code_in_codebook(a))
A.add_codes([(b, c)])
self.assertTrue(A._code_in_codebook(b))
self.assertTrue(A._code_in_codebook(c))
A.add_codes([(d, c)])
self.assertTrue(A._code_in_codebook(d))
self.assertTrue(A._code_in_codebook(c))
A.add_codes([(e, None)])
self.assertTrue(A._code_in_codebook(e))
self.assertRaises(ValueError, A.add_codes, [e])
self.assertRaises(ValueError, A.add_codes, [(e, b)])
def test_cache_labels_language(self):
"""Does caching labels for multiple language work
esp. caching non-existence of a label"""
from amcat.models.language import Language
l1 = Language.objects.get(pk=1)
a = amcattest.create_test_code(label="a", language=l1)
l2 = Language.objects.get(pk=2)
b = amcattest.create_test_code(label="b", language=l2)
A = amcattest.create_test_codebook(name="A")
A.add_code(a); A.add_code(b)
with self.checkMaxQueries(3, "Cache labels"):
# 1 for "normal" caching, and 1 for each cache_labels()
A.cache_labels(l1)
A.cache_labels(l2)
A.invalidate_cache()
with self.checkMaxQueries(2, "Cache labels"):
A.cache_labels(l1, l2)
with self.checkMaxQueries(0, "Get cached codes"):
a, b = map(A.get_code, [a.id, b.id])
with self.checkMaxQueries(0, "Get multiple cached codes"):
list(A.get_codes())
with self.checkMaxQueries(0, "Get exisitng labels"):
self.assertEqual(a.get_label(l1), "a")
self.assertEqual(b.get_label(l2), "b")
with self.checkMaxQueries(0, "Get non-existing labels"):
self.assertEqual(a.get_label(l2, l1), "a")
self.assertEqual(b.get_label(l1, l2), "b")
with self.checkMaxQueries(0, "Get tree"):
list(A.get_tree())
c = amcattest.create_test_code(label="c", language=l2)
with self.checkMaxQueries(5, "Add new code"):
A.add_code(c)
def test_rules(self):
cb = amcattest.create_test_codebook()
lang = amcattest.get_test_language()
r = RuleSet.objects.create(label="test", lexicon_codebook=cb,
lexicon_language=lang)
condition = "?x :rel_nsubj ?y"
insert = "?x :boe ?y"
Rule.objects.create(ruleset=r, label="x", order=2,
where=condition, insert=insert)
getrules = lambda r : [{k:v for k,v in rule.iteritems()
if k in ["condition", "insert"]}
for rule in r.get_rules()]
self.assertEqual(getrules(r),
[{"condition": condition, "insert": insert}])
Rule.objects.create(ruleset=r, label="y", order=1,
where="w", insert="i")
self.assertEqual(getrules(r),
[{"condition": "w", "insert": "i"},
{"condition": condition, "insert": insert}])