def query_list_to_table(queries, maxqlen=80, output=False, normalise_numbers=True, **outputoptions):
"""Convert a django query list (list of dict with keys time and sql) into a table3
If output is non-False, output the table with the given options
Specify print, "print", or a stream for output to be printed immediately
"""
time = collections.defaultdict(list)
for q in queries:
query = q["sql"]
if normalise_numbers:
query = re.sub(r"\d+", "#", query)
#print(query)
time[query].append(float(q["time"]))
t = ObjectTable(rows = time.items())
t.addColumn(lambda (k, v) : len(v), "N")
t.addColumn(lambda (k, v) : k[:maxqlen], "Query")
cum = t.addColumn(lambda (k, v): "%1.4f" % sum(v), "Cum.")
t.addColumn(lambda (k, v): "%1.4f" % (sum(v) / len(v)), "Avg.")
t = SortedTable(t, sort=cum)
if output:
if "stream" not in outputoptions and output is not True:
if output in (print, "print"):
import sys
outputoptions["stream"] = sys.stdout
else:
outputoptions["stream"] = output
t.output(**outputoptions)
return t
def query_list_to_table(queries, maxqlen=120, output=False, normalise_numbers=True, **outputoptions):
"""Convert a django query list (list of dict with keys time and sql) into a table3
If output is non-False, output the table with the given options
Specify print, "print", or a stream for output to be printed immediately
"""
time = collections.defaultdict(list)
for q in queries:
query = q["sql"]
if normalise_numbers:
query = re.sub(r"\d+", "#", query)
#print(query)
time[query].append(float(q["time"]))
t = ObjectTable(rows = time.items())
t.addColumn(lambda (k, v) : len(v), "N")
t.addColumn(lambda (k, v) : k[:maxqlen], "Query")
cum = t.addColumn(lambda (k, v): "%1.4f" % sum(v), "Cum.")
t.addColumn(lambda (k, v): "%1.4f" % (sum(v) / len(v)), "Avg.")
t = SortedTable(t, sort=cum)
if output:
if "stream" not in outputoptions and output is not True:
if output in (print, "print"):
import sys
outputoptions["stream"] = sys.stdout
else:
outputoptions["stream"] = output
t.output(**outputoptions)
return t
def test_object_table(self):
"""Does creating object tables work"""
class Test(object):
def __init__(self, a, b, c):
self.a, self.b, self.c = a, b, c
l = ObjectTable(
rows=[Test(1, 2, 3),
Test("bla", None, 7),
Test(-1, -1, None)])
l.add_column(lambda x: x.a, "de a")
l.add_column("b")
l.add_column(ObjectColumn("en de C", lambda x: x.c))
result = tableoutput.table2unicode(l)
# get rid of pesky unicode
result = result.translate(
dict((a, 65 + a % 26) for a in range(0x2500, 0x2600)))
correct = '''OKKKKKKEKKKKEKKKKKKKKKR
L de a K b K en de C L
ZIIIIIIQIIIIQIIIIIIIIIC
L 1 K 2 K 3 L
L bla K K 7 L
L -1 K -1 K L
UKKKKKKHKKKKHKKKKKKKKKX'''
self.assertEquals(_striplines(result), _striplines(correct.strip()))
def run(self, _input=None):
c = get_codebook(self.options["codebook"].id)
for lang in LABEL_LANGUAGE + QUERY_LANGUAGE:
c.cache_labels(lang)
t = ObjectTable(rows=c.get_codes())
t.addColumn(lambda c: c.id, "id")
t.addColumn(lambda c: c.get_label(*LABEL_LANGUAGE), "label")
t.addColumn(lambda c: c.get_label(*QUERY_LANGUAGE, fallback=False),
"query")
return t
def get_table_sentence_codings_article(codedarticle, language):
"""Return a table of sentence codings x fields
The cells contain domain (deserialized) objects
"""
result = ObjectTable(rows = list(codedarticle.sentence_codings.prefetch_related("values")))
result.addColumn('id')
result.addColumn(lambda x:x.sentence_id, 'sentence')
for field in codedarticle.codingjob.unitschema.fields.select_related("fieldtype").order_by('fieldnr').all():
result.addColumn(CodingColumn(field, language))
return result
def run(self, _input=None):
c = self.options["codebook"]
for lang in LABEL_LANGUAGE + QUERY_LANGUAGE:
c.cache_labels(lang)
t = ObjectTable(rows=c.get_codes())
t.addColumn(lambda c: c.id, "id")
t.addColumn(lambda c: c.get_label(*LABEL_LANGUAGE), "label")
t.addColumn(lambda c: c.get_label(*QUERY_LANGUAGE, fallback=False), "query")
return t
def get_table_sentence_codings_article(codedarticle, language):
"""Return a table of sentence codings x fields
The cells contain domain (deserialized) objects
"""
#import pdb
#pdb.set_trace()
result = ObjectTable(rows=list(codedarticle.sentence_codings))
result.addColumn('id')
result.addColumn(lambda x: x.sentence_id, 'sentence')
for field in codedarticle.codingjob.unitschema.fields.order_by(
'fieldnr').all():
result.addColumn(partial(get_label, field, language), field.label)
return result
def index(request):
daily_scrapers = list(Scraper.objects.filter(run_daily=True, active=True))
non_daily_scrapers = list(
Scraper.objects.filter(run_daily=False, active=True))
inactive_scrapers = list(Scraper.objects.filter(active=False))
dates = [
datetime.date.today() - datetime.timedelta(days=n) for n in range(14)
]
scraper_lists = {
"daily_table": daily_scrapers,
"non_daily_table": non_daily_scrapers,
"inactive_table": inactive_scrapers
}
for s_list in scraper_lists.values():
for scraper in s_list:
scraper.articles = scraper.n_scraped_articles(from_date=dates[-1],
to_date=dates[0])
scraper_tables = {
name: ObjectTable(rows=s_list, columns=["id", "label"])
for name, s_list in scraper_lists.items()
}
def Set(scraper):
s = scraper.articleset
if s is None: return ""
url = reverse(articleset, args=[s.project.id, s.id])
return "<a href='{url}'>{s}</a>".format(**locals())
for s_table in scraper_tables.values():
s_table.addColumn(Set)
def getdate(date, scraper):
return scraper.articles.get(date, 0)
for date in dates:
for s_table in scraper_tables.values():
s_table.addColumn(partial(getdate, date), str(date)[-5:])
table_dict = {}
for t_name, s_table in scraper_tables.items():
table_dict[t_name] = table2htmlDjango(s_table, safe=True)
return render(request, 'navigator/scrapers/index.html',
dict(locals().items() + table_dict.items()))
def test_object_table(self):
"""Does creating object tables work"""
class Test(object):
def __init__(self, a, b, c):
self.a, self.b, self.c = a, b, c
l = ObjectTable(rows=[Test(1, 2, 3), Test("bla", None, 7), Test(-1, -1, None)])
l.addColumn(lambda x: x.a, "de a")
l.addColumn("b")
l.addColumn(ObjectColumn("en de C", lambda x: x.c))
result = tableoutput.table2unicode(l)
# get rid of pesky unicode
result = result.translate(dict((a, 65 + a % 26) for a in range(0x2500, 0x2600)))
correct = '''OKKKKKKEKKKKEKKKKKKKKKR
L de a K b K en de C L
ZIIIIIIQIIIIQIIIIIIIIIC
L 1 K 2 K 3 L
L bla K K 7 L
L -1 K -1 K L
UKKKKKKHKKKKHKKKKKKKKKX'''
self.assertEquals(_striplines(result), _striplines(correct.strip()))
def get_table_articles_per_job(jobs):
"""Return a table of all articles in a cjset with status
Columns: article, articlemeta, status, comments
"""
result = ObjectTable(rows=list(
get_coded_articles(
jobs, cache_coding=True, select_related=("medium", ))))
result.addColumn(lambda a: a.article.id, "articleid")
result.addColumn(lambda a: a.article.headline, "headline")
result.addColumn(lambda a: a.article.date, "date")
result.addColumn(lambda a: a.article.medium, "medium")
result.addColumn(lambda a: a.article.pagenr, "pagenr")
result.addColumn(lambda a: a.article.length, "length")
result.addColumn(lambda a: a.coding and a.coding.status, "status")
result.addColumn(lambda a: a.coding and a.coding.comments, "comments")
return result
def get_table_jobs_per_user(users, **additionalFilters):
"""Return a table of all jobs per user
Columns: ids, jobname, coder, issuer, issuedate, #articles, #completed, #inprogress
"""
try:
iter(users)
except TypeError:
users = [users]
jobs = list(
CodingJob.objects.filter(coder__in=users,
**additionalFilters).order_by("-id"))
result = ObjectTable(rows=jobs)
result.addColumn("id")
result.addColumn("name")
result.addColumn("coder")
result.addColumn(lambda s: s.insertuser, "insertuser")
result.addColumn(lambda s: s.insertdate, "insertdate")
result.addColumn(lambda s: s.articleset.articles.count(), "narticles")
result.addColumn(
lambda s: s.codings.filter(sentence=None, status=2).count(),
"ncomplete")
return result
def get_table_articles_per_job(jobs):
"""Return a table of all articles in a cjset with status
Columns: article, articlemeta, status, comments
"""
result = ObjectTable(
rows=list(get_coded_articles(
jobs, cache_coding=True, select_related=("medium",)
))
)
result.addColumn(lambda a: a.article.id, "articleid")
result.addColumn(lambda a: a.article.headline, "headline")
result.addColumn(lambda a: a.article.date, "date")
result.addColumn(lambda a: a.article.medium, "medium")
result.addColumn(lambda a: a.article.pagenr, "pagenr")
result.addColumn(lambda a: a.article.length, "length")
result.addColumn(lambda a: a.coding and a.coding.status, "status")
result.addColumn(lambda a: a.coding and a.coding.comments, "comments")
return result
def get_table_jobs_per_user(users, **additionalFilters):
"""Return a table of all jobs per user
Columns: ids, jobname, coder, issuer, issuedate, #articles, #completed, #inprogress
"""
try:
# Workaround: iter(x) needs to be evaluated. while LazyObjects yield only one
# element even though iter() is called.
list(iter(users))
except TypeError:
users = [users]
jobs = list(CodingJob.objects.filter(coder__in=users, **additionalFilters).order_by("-id"))
result = ObjectTable(rows=jobs)
result.addColumn("id")
result.addColumn("name")
result.addColumn("coder")
result.addColumn(lambda s: s.insertuser, "insertuser")
result.addColumn(lambda s :s.insertdate, "insertdate")
result.addColumn(lambda s :s.articleset.articles.count(), "narticles")
result.addColumn(lambda s :s.codings.filter(sentence=None, status=2).count(),
"ncomplete")
return result
