def query_list_to_table(queries, maxqlen=120, output=False, normalise_numbers=True, **outputoptions):
"""Convert a django query list (list of dict with keys time and sql) into a table3
If output is non-False, output the table with the given options
Specify print, "print", or a stream for output to be printed immediately
"""
time = collections.defaultdict(list)
for q in queries:
query = q["sql"]
if normalise_numbers:
query = re.sub(r"\d+", "#", query)
#print(query)
time[query].append(float(q["time"]))
t = ObjectTable(rows = time.items())
t.addColumn(lambda (k, v) : len(v), "N")
t.addColumn(lambda (k, v) : k[:maxqlen], "Query")
cum = t.addColumn(lambda (k, v): "%1.4f" % sum(v), "Cum.")
t.addColumn(lambda (k, v): "%1.4f" % (sum(v) / len(v)), "Avg.")
t = SortedTable(t, sort=cum)
if output:
if "stream" not in outputoptions and output is not True:
if output in (print, "print"):
import sys
outputoptions["stream"] = sys.stdout
else:
outputoptions["stream"] = output
t.output(**outputoptions)
return t
def test_object_table(self):
"""Does creating object tables work"""
class Test(object):
def __init__(self, a, b, c):
self.a, self.b, self.c = a, b, c
l = ObjectTable(
rows=[Test(1, 2, 3),
Test("bla", None, 7),
Test(-1, -1, None)])
l.addColumn(lambda x: x.a, "de a")
l.addColumn("b")
l.addColumn(ObjectColumn("en de C", lambda x: x.c))
result = tableoutput.table2unicode(l)
# get rid of pesky unicode
result = result.translate(
dict((a, 65 + a % 26) for a in range(0x2500, 0x2600)))
correct = '''OKKKKKKEKKKKEKKKKKKKKKR
L de a K b K en de C L
ZIIIIIIQIIIIQIIIIIIIIIC
L 1 K 2 K 3 L
L bla K K 7 L
L -1 K -1 K L
UKKKKKKHKKKKHKKKKKKKKKX'''
self.assertEquals(_striplines(result), _striplines(correct.strip()))
def run(self, _input=None):
c = self.options["codebook"]
for lang in LABEL_LANGUAGE + QUERY_LANGUAGE:
c.cache_labels(lang)
t = ObjectTable(rows=c.get_codes())
t.addColumn(lambda c: c.id, "id")
t.addColumn(lambda c: c.get_label(*LABEL_LANGUAGE), "label")
t.addColumn(lambda c: c.get_label(*QUERY_LANGUAGE, fallback=False), "query")
return t
def run(self, _input=None):
c = get_codebook(self.options["codebook"].id)
for lang in LABEL_LANGUAGE + QUERY_LANGUAGE:
c.cache_labels(lang)
t = ObjectTable(rows=c.get_codes())
t.addColumn(lambda c: c.id, "id")
t.addColumn(lambda c: c.get_label(*LABEL_LANGUAGE), "label")
t.addColumn(lambda c: c.get_label(*QUERY_LANGUAGE, fallback=False),
"query")
return t
def get_table_sentence_codings_article(codedarticle, language):
"""Return a table of sentence codings x fields
The cells contain domain (deserialized) objects
"""
result = ObjectTable(rows = list(codedarticle.sentence_codings.prefetch_related("values")))
result.addColumn('id')
result.addColumn(lambda x:x.sentence_id, 'sentence')
for field in codedarticle.codingjob.unitschema.fields.select_related("fieldtype").order_by('fieldnr').all():
result.addColumn(CodingColumn(field, language))
return result
def get_table_sentence_codings_article(codedarticle, language):
"""Return a table of sentence codings x fields
The cells contain domain (deserialized) objects
"""
#import pdb
#pdb.set_trace()
result = ObjectTable(rows=list(codedarticle.sentence_codings))
result.addColumn('id')
result.addColumn(lambda x: x.sentence_id, 'sentence')
for field in codedarticle.codingjob.unitschema.fields.order_by(
'fieldnr').all():
result.addColumn(partial(get_label, field, language), field.label)
return result
def test_object_table(self):
"""Does creating object tables work"""
class Test(object):
def __init__(self, a, b, c):
self.a, self.b, self.c = a, b, c
l = ObjectTable(rows=[Test(1, 2, 3), Test("bla", None, 7), Test(-1, -1, None)])
l.addColumn(lambda x: x.a, "de a")
l.addColumn("b")
l.addColumn(ObjectColumn("en de C", lambda x: x.c))
result = tableoutput.table2unicode(l)
# get rid of pesky unicode
result = result.translate(dict((a, 65 + a % 26) for a in range(0x2500, 0x2600)))
correct = '''OKKKKKKEKKKKEKKKKKKKKKR
L de a K b K en de C L
ZIIIIIIQIIIIQIIIIIIIIIC
L 1 K 2 K 3 L
L bla K K 7 L
L -1 K -1 K L
UKKKKKKHKKKKHKKKKKKKKKX'''
self.assertEquals(_striplines(result), _striplines(correct.strip()))
def get_table_articles_per_job(jobs):
"""Return a table of all articles in a cjset with status
Columns: article, articlemeta, status, comments
"""
result = ObjectTable(rows=list(
get_coded_articles(
jobs, cache_coding=True, select_related=("medium", ))))
result.addColumn(lambda a: a.article.id, "articleid")
result.addColumn(lambda a: a.article.headline, "headline")
result.addColumn(lambda a: a.article.date, "date")
result.addColumn(lambda a: a.article.medium, "medium")
result.addColumn(lambda a: a.article.pagenr, "pagenr")
result.addColumn(lambda a: a.article.length, "length")
result.addColumn(lambda a: a.coding and a.coding.status, "status")
result.addColumn(lambda a: a.coding and a.coding.comments, "comments")
return result
def get_table_jobs_per_user(users, **additionalFilters):
"""Return a table of all jobs per user
Columns: ids, jobname, coder, issuer, issuedate, #articles, #completed, #inprogress
"""
try:
iter(users)
except TypeError:
users = [users]
jobs = list(
CodingJob.objects.filter(coder__in=users,
**additionalFilters).order_by("-id"))
result = ObjectTable(rows=jobs)
result.addColumn("id")
result.addColumn("name")
result.addColumn("coder")
result.addColumn(lambda s: s.insertuser, "insertuser")
result.addColumn(lambda s: s.insertdate, "insertdate")
result.addColumn(lambda s: s.articleset.articles.count(), "narticles")
result.addColumn(
lambda s: s.codings.filter(sentence=None, status=2).count(),
"ncomplete")
return result
def get_table_articles_per_job(jobs):
"""Return a table of all articles in a cjset with status
Columns: article, articlemeta, status, comments
"""
result = ObjectTable(
rows=list(get_coded_articles(
jobs, cache_coding=True, select_related=("medium",)
))
)
result.addColumn(lambda a: a.article.id, "articleid")
result.addColumn(lambda a: a.article.headline, "headline")
result.addColumn(lambda a: a.article.date, "date")
result.addColumn(lambda a: a.article.medium, "medium")
result.addColumn(lambda a: a.article.pagenr, "pagenr")
result.addColumn(lambda a: a.article.length, "length")
result.addColumn(lambda a: a.coding and a.coding.status, "status")
result.addColumn(lambda a: a.coding and a.coding.comments, "comments")
return result
def get_table_jobs_per_user(users, **additionalFilters):
"""Return a table of all jobs per user
Columns: ids, jobname, coder, issuer, issuedate, #articles, #completed, #inprogress
"""
try:
# Workaround: iter(x) needs to be evaluated. while LazyObjects yield only one
# element even though iter() is called.
list(iter(users))
except TypeError:
users = [users]
jobs = list(CodingJob.objects.filter(coder__in=users, **additionalFilters).order_by("-id"))
result = ObjectTable(rows=jobs)
result.addColumn("id")
result.addColumn("name")
result.addColumn("coder")
result.addColumn(lambda s: s.insertuser, "insertuser")
result.addColumn(lambda s :s.insertdate, "insertdate")
result.addColumn(lambda s :s.articleset.articles.count(), "narticles")
result.addColumn(lambda s :s.codings.filter(sentence=None, status=2).count(),
"ncomplete")
return result
