def run(self):
sf = SelectionForm(self.project, self.data)
sf.full_clean()
article_ids = list(keywordsearch.get_ids(sf.cleaned_data))
result = SaveAsSetScript(self.data).run(article_ids)
result.provenance = json.dumps(dict(self.data))
result.save()
return self.outputResponse(result, object)
def run(self):
sel = SelectionForm(project=self.project, data=self.data)
if not sel.is_valid():
# This should not happen when using normal pages (i.e., someone is trying
# to hack)
forms.ValidationError("Non-valid values entered.")
sel.full_clean()
if sel.use_solr:
sel.cleaned_data['length'] = 99999999 # Return 'unlimited' results
sel.cleaned_data['sortColumn'] = None
sel.cleaned_data['start'] = 0
sel.cleaned_data['columns'] = []
articles = solrlib.getArticles(sel.cleaned_data)
else:
articles = database.get_queryset(**sel.cleaned_data)
project = Project.objects.get(id=self.data.get('projects'))
# Create articleset
a = ArticleSet.objects.create(project=project,
name=self.data['setname'])
a.add(*articles)
# Split all articles
CreateSentences(dict(articlesets=[a.id])).run()
# Create codingjob
coder = User.objects.get(id=self.data['coder'])
articleschema = CodingSchema.objects.get(id=self.data['articleschema'])
unitschema = CodingSchema.objects.get(id=self.data['unitschema'])
if not 'insertuser' in self.data:
insertuser = auth.get_request().user
else:
insertuser = User.objects.get(id=self.data['insertuser'])
c = CodingJob.objects.create(project=project,
name=self.data['setname'],
articleset=a,
coder=coder,
articleschema=articleschema,
unitschema=unitschema,
insertuser=insertuser)
html = "<div>Saved as <a href='%s'>coding job %s</a>.</div>"
return HttpResponse(json.dumps({
"html":
html % (reverse("codingjob", args=[project.id, c.id]), c.id),
"webscriptClassname":
self.__class__.__name__,
"webscriptName":
self.name,
"doNotAddActionToMainForm":
True
}),
mimetype='application/json')
def run(self):
self.progress_monitor.update(1, "Listing articles")
sf = SelectionForm(self.project, self.data)
sf.full_clean()
article_ids = list(keywordsearch.get_ids(sf.cleaned_data))
self.progress_monitor.update(39, "Creating set with {n} articles".format(n=len(article_ids)))
result = SaveAsSetScript(self.data, monitor=self.progress_monitor).run(article_ids)
result.provenance = json.dumps(dict(self.data))
result.save()
return self.outputResponse(result, object)
def run(self):
self.progress_monitor.update(1, "Listing articles")
sf = SelectionForm(self.project, self.data)
sf.full_clean()
article_ids = list(keywordsearch.get_ids(sf.cleaned_data))
self.progress_monitor.update(
39, "Creating set with {n} articles".format(n=len(article_ids)))
result = SaveAsSetScript(
self.data, monitor=self.progress_monitor).run(article_ids)
result.provenance = json.dumps(dict(self.data))
result.save()
return self.outputResponse(result, object)
def run(self):
selection = SelectionForm(project=self.project, data=self.data)
selection.full_clean()
aggrTable = AggregationScript(project=self.project,
options=self.data).run()
if self.output == 'json-html' or (self.output == 'html' and
self.options['graphOnly'] == True):
columns = sorted(aggrTable.getColumns(),
key=lambda x: x.id if hasattr(x, 'id') else x)
dataTablesHeaderDict = [{'mDataProp':TITLE_COLUMN_NAME,'sTitle':TITLE_COLUMN_NAME, 'sType':'objid', 'sWidth':'100px'}] + \
[{'mDataProp':get_key(col),'mData':get_key(col),'sTitle':col, 'sWidth':'70px'} for col in columns]
dataJson = []
for row in aggrTable.getRows():
rowJson = dict([(get_key(col), aggrTable.getValue(row, col))
for col in columns])
rowJson[TITLE_COLUMN_NAME] = row
dataJson.append(rowJson)
if self.options['xAxis'] == 'date':
datesDict = dict(
_getDatesDict(aggrTable, self.options['dateInterval']))
else:
datesDict = {}
labels = {q.label: q.query for q in aggrTable.queries}
aggregationType = 'hits' if self.options[
'counterType'] == 'numberOfHits' else 'articles'
graphOnly = 'true' if self.options['graphOnly'] == True else 'false'
scriptoutput = render_to_string(
'api/webscripts/aggregation.html', {
'dataJson': json.dumps(dataJson),
'columnsJson': json.dumps(dataTablesHeaderDict),
'aggregationType': aggregationType,
'datesDict': json.dumps(datesDict),
'graphOnly': graphOnly,
'labels': json.dumps(labels),
'ownForm': self.form(project=self.project, data=self.data),
'relative': int(self.options['relative'])
})
if self.output == 'json-html':
return self.outputJsonHtml(scriptoutput)
else:
return HttpResponse(scriptoutput, mimetype='text/html')
else: # json-html output
return self.outputResponse(aggrTable,
AggregationScript.output_type)
def run(self):
sel = SelectionForm(project=self.project, data=self.data)
if not sel.is_valid():
# This should not happen when using normal pages (i.e., someone is trying
# to hack)
forms.ValidationError("Non-valid values entered.")
articles = list(keywordsearch.get_ids(self.data))
# Create articleset
a = ArticleSet.create_set(project=self.project,
articles=articles,
name=self.data['setname'],
favourite=False)
# Split all articles
CreateSentences(dict(articlesets=[a.id])).run()
# Create codingjob
coder = User.objects.get(id=self.data['coder'])
articleschema = CodingSchema.objects.get(id=self.data['articleschema'])
unitschema = CodingSchema.objects.get(id=self.data['unitschema'])
if not 'insertuser' in self.data:
insertuser = auth.get_request().user
else:
insertuser = User.objects.get(id=self.data['insertuser'])
c = CodingJob.objects.create(project=self.project,
name=self.data['setname'],
articleset=a,
coder=coder,
articleschema=articleschema,
unitschema=unitschema,
insertuser=insertuser)
html = "<div>Saved as <a href='%s'>coding job %s</a>.</div>"
return HttpResponse(json.dumps({
"html":
html % (reverse("coding job-details", args=[self.project.id, c.id
]), c.id),
"webscriptClassname":
self.__class__.__name__,
"webscriptName":
self.name,
"doNotAddActionToMainForm":
True
}),
mimetype='application/json')
def outputJsonHtml(self, scriptoutput):
actions = self.getActions()
webscriptform = None
if self.form:
try:
webscriptform = self.form(project=self.project, data=self.data)
except TypeError:
webscriptform = self.form(data=self.data)
selectionform = SelectionForm(project=self.project, data=self.data)
html = render_to_string('navigator/selection/webscriptoutput.html', {
'scriptoutput':scriptoutput,
'actions': actions,
'selectionform':selectionform,
'webscriptform':webscriptform
})
reply = {}
reply['html'] = html
reply['queries'] = getQueries()
reply['querytime'] = '%.2f' % (time.time() - self.initTime)
reply['webscriptName'] = self.name
reply['webscriptClassname'] = self.__class__.__name__
jsondata = json.dumps(reply, default=lambda o:unicode(o))
return HttpResponse(jsondata, mimetype='text/plain')
def _run_query(self, form_data, expected_indices=None, expected_count=None, msg=None):
self._setUp()
sets = ArticleSet.objects.filter(pk=self.articleset.pk)
form = SelectionForm(articlesets=sets, project=self.articleset.project, data=form_data)
form.full_clean()
self.assertFalse(form.errors, "Form contains errors")
search = SelectionSearch(form)
if expected_indices:
article_ids = search.get_article_ids()
articles = Article.objects.filter(id__in=article_ids)
expected = [self.articles[i] for i in expected_indices]
self.assertSetEqual(set(articles), set(expected), msg=msg)
if expected_count:
self.assertEqual(search.get_count(), expected_count, msg=msg)
def index(request):
"""returns the selection form"""
outputs = []
for ws in mainScripts:
outputs.append({'id':ws.__name__, 'name':ws.name, 'formAsHtml': ws.formHtml()})
context = {'form':SelectionForm(data=request.REQUEST), 'outputs':outputs}
return render(request, 'navigator/selection/form.html', context)
def run(self):
selection = SelectionForm(project=self.project, data=self.data)
selection.full_clean()
aggrTable = AggregationScript(project=self.project, options=self.data, monitor=self.progress_monitor).run()
if self.output == 'json-html' or (self.output == 'html' and self.options['graphOnly'] == True):
columns = sorted(aggrTable.getColumns(), key=lambda x:x.id if hasattr(x,'id') else x)
dataTablesHeaderDict = [{'mDataProp':TITLE_COLUMN_NAME,'sTitle':TITLE_COLUMN_NAME, 'sType':'objid', 'sWidth':'100px'}] + \
[{'mDataProp':get_key(col),'mData':get_key(col),'sTitle':col, 'sWidth':'70px'} for col in columns]
dataJson = []
for row in aggrTable.getRows():
rowJson = dict([(get_key(col), aggrTable.getValue(row, col)) for col in columns])
rowJson[TITLE_COLUMN_NAME] = row
dataJson.append(rowJson)
if self.options['xAxis'] == 'date':
datesDict = dict(_getDatesDict(aggrTable, self.options['dateInterval']))
else:
datesDict = {}
labels = {q.label : q.query for q in aggrTable.queries}
aggregationType = 'articles'
graphOnly = 'true' if self.options['graphOnly'] == True else 'false'
scriptoutput = render_to_string('api/webscripts/aggregation.html', {
'dataJson':json.dumps(dataJson),
'columnsJson':json.dumps(dataTablesHeaderDict),
'aggregationType':aggregationType,
'datesDict': json.dumps(datesDict),
'graphOnly': graphOnly,
'labels' : json.dumps(labels),
'ownForm':self.form(project=self.project, data=self.data),
'relative':int(self.options['relative'])
})
if self.output == 'json-html':
return self.outputJsonHtml(scriptoutput)
else:
return HttpResponse(scriptoutput, mimetype='text/html')
else: # json-html output
return self.outputResponse(aggrTable, AggregationScript.output_type)
def run(self):
formData = self.data.copy() # copy needed since formData is inmutable
if "articlesets" not in formData:
artsets = [str(aset.id) for aset in Project.objects.get(id=formData['projects']).all_articlesets()]
formData.setlist("articlesets", artsets)
if isinstance(self.data['projects'], (basestring, int)):
project_id = int(self.data['projects'])
else:
project_id = int(self.data['projects'][0])
formData["start_date"] = formData["start_date"].split("T")[0]
formData["end_date"] = formData["end_date"].split("T")[0]
sf = SelectionForm(self.project, formData)
if not sf.is_valid():
raise ValueError(dict(sf._errors))
t = keywordsearch.getDatatable(sf.cleaned_data, rowlink_open_in="new", allow_html_export=True)
t = t.rowlink_reverse("project-article-details", args=[project_id, '{id}'])
cols = {FORM_FIELDS_TO_ELASTIC.get(f, f) for f in self.data.getlist('columns')}
for f in list(t.get_fields()):
if f not in cols:
t = t.hide(f)
if 'kwic' in cols and not self.data.get('query'):
raise Exception("Cannot provide Keyword in Context without query")
for col in cols & {'hits', 'text', 'lead', 'kwic'}:
t = t.add_arguments(col=col)
html = unicode(t)
stats_log.info(json.dumps({
"actions": "query:articlelist", "user": self.user.username,
"project_id": self.project.id, "project__name": self.project.name
}))
if self.output == "html":
response = HttpResponse(mimetype='text/html')
response.write(html)
return response
else:
return self.outputJsonHtml(html)
def run(self):
self.progress_monitor.update(1, "Creating summary")
if isinstance(self.data['projects'], (basestring, int)):
project_id = int(self.data['projects'])
else:
project_id = int(self.data['projects'][0])
sf = SelectionForm(self.project, self.data)
sf.full_clean()
n = keywordsearch.get_total_n(sf.cleaned_data)
self.progress_monitor.update(39, "Found {n} articles in total".format(**locals()))
articles = list(ArticleListScript(self.data).run())
for a in articles:
a.hack_project_id = project_id
self.output_template = 'api/webscripts/articlelist.html'
self.progress_monitor.update(40, "Created summary")
return self.outputResponse(dict(articlelist=articles, n=n, page=self.data.get('start')), ArticleListScript.output_type)
def run(self):
sel = SelectionForm(project=self.project, data=self.data)
if not sel.is_valid():
# This should not happen when using normal pages (i.e., someone is trying
# to hack)
forms.ValidationError("Non-valid values entered.")
sel.full_clean()
if sel.use_solr:
sel.cleaned_data['length'] = 99999999 # Return 'unlimited' results
sel.cleaned_data['sortColumn'] = None
sel.cleaned_data['start'] = 0
sel.cleaned_data['columns'] = []
articles = solrlib.getArticles(sel.cleaned_data)
else:
articles = database.get_queryset(**sel.cleaned_data)
project = Project.objects.get(id=self.data.get('projects'))
# Create articleset
a = ArticleSet.objects.create(project=project, name=self.data['setname'])
a.add(*articles)
# Split all articles
CreateSentences(dict(articlesets=[a.id])).run()
# Create codingjob
coder = User.objects.get(id=self.data['coder'])
articleschema = CodingSchema.objects.get(id=self.data['articleschema'])
unitschema = CodingSchema.objects.get(id=self.data['unitschema'])
if not 'insertuser' in self.data:
insertuser = auth.get_request().user
else:
insertuser = User.objects.get(id=self.data['insertuser'])
c = CodingJob.objects.create(project=project, name=self.data['setname'], articleset=a,
coder=coder, articleschema=articleschema, unitschema=unitschema,
insertuser=insertuser)
html = "<div>Saved as <a href='%s'>coding job %s</a>.</div>"
return HttpResponse(json.dumps({
"html" : html % (reverse("codingjob", args=[project.id, c.id]), c.id),
"webscriptClassname" : self.__class__.__name__,
"webscriptName" : self.name,
"doNotAddActionToMainForm" : True
}), mimetype='application/json')
def run(self):
sel = SelectionForm(project=self.project, data=self.data)
if not sel.is_valid():
# This should not happen when using normal pages (i.e., someone is trying
# to hack)
forms.ValidationError("Non-valid values entered.")
articles = list(keywordsearch.get_ids(self.data))
# Create articleset
a = ArticleSet.create_set(project=self.project, articles=articles, name=self.data['setname'], favourite=False)
# Split all articles
CreateSentences(dict(articlesets=[a.id])).run()
# Create codingjob
coder = User.objects.get(id=self.data['coder'])
articleschema = CodingSchema.objects.get(id=self.data['articleschema'])
unitschema = CodingSchema.objects.get(id=self.data['unitschema'])
if not 'insertuser' in self.data:
insertuser = auth.get_request().user
else:
insertuser = User.objects.get(id=self.data['insertuser'])
c = CodingJob.objects.create(project=self.project, name=self.data['setname'], articleset=a,
coder=coder, articleschema=articleschema, unitschema=unitschema,
insertuser=insertuser)
html = "<div>Saved as <a href='%s'>coding job %s</a>.</div>"
return HttpResponse(json.dumps({
"html" : html % (reverse("coding job-details", args=[self.project.id, c.id]), c.id),
"webscriptClassname" : self.__class__.__name__,
"webscriptName" : self.name,
"doNotAddActionToMainForm" : True
}), mimetype='application/json')
def selection(request, project):
"""
Render article selection page.
TODO:
- update to meet PEP8 style
- remove/replace webscripts (?)
"""
outputs = []
for ws in mainScripts:
outputs.append({
'id': ws.__name__,
'name': ws.name,
'formAsHtml': ws.formHtml(project=project)
})
all_articlesets = project.all_articlesets()
favs = tuple(
project.favourite_articlesets.filter(
Q(project=project.id) | Q(projects_set=project.id)).values_list(
"id", flat=True))
no_favourites = not favs
favourites = json.dumps(favs)
codingjobs = json.dumps(
tuple(
CodingJob.objects.filter(
articleset__in=all_articlesets).values_list("articleset_id",
flat=True)))
all_sets = json.dumps(tuple(all_articlesets.values_list("id", flat=True)))
ctx = locals()
ctx.update({
'form':
SelectionForm(project=project,
data=request.GET,
initial={"datetype": "all"}),
'outputs':
outputs,
'project':
project,
'context':
project,
'menu':
PROJECT_MENU,
'selected':
'query'
})
return render(request, 'navigator/project/selection.html', ctx)
def get_context_data(self, **kwargs):
articleset_ids = set(
map(
int,
filter(unicode.isdigit,
self.request.GET.get("sets", "").split(","))))
articlesets = self.project.all_articlesets().filter(
id__in=articleset_ids)
articlesets = articlesets.only("id", "name")
query_id = self.request.GET.get("query", "null")
user_id = self.request.user.id
articleset_ids_json = json.dumps(list(articleset_ids))
codebooks = self.project.get_codebooks().order_by("name").only(
"id", "name")
all_articlesets = self.project.all_articlesets().filter(
codingjob_set__id__isnull=True).only("id", "name")
saved_queries_table = self.get_saved_queries_table()
articlesets_table = self.get_articlesets_table()
form = SelectionForm(project=self.project,
articlesets=articlesets,
data=self.request.GET,
initial={
"datetype": "all",
"articlesets": articlesets
})
statistics = amcates.ES().statistics(
filters={"sets": list(articleset_ids)})
settings = conf.settings
saved_queries = Query.objects.filter(project=self.project)
saved_user_queries = saved_queries.filter(
user=self.request.user)[:SHOW_N_RECENT_QUERIES]
saved_project_queries = saved_queries.filter(~Q(
user=self.request.user))[:SHOW_N_RECENT_QUERIES]
form.fields["articlesets"].widget.attrs['disabled'] = 'disabled'
return dict(super(QueryView, self).get_context_data(), **locals())
def get_context_data(self, **kwargs):
context = super(QueryView, self).get_context_data(**kwargs)
p = self.project
outputs = [
dict(id=ws.__name__,
name=ws.name,
formAsHtml=ws.formHtml(project=p)) for ws in mainScripts
]
all_articlesets = p.all_articlesets()
initial = {
int(aset)
for aset in self.request.GET.getlist("articlesets")
if aset.isnumeric()
}
initial = initial.intersection(
set(all_articlesets.values_list("id", flat=True)))
favs = set(
p.favourite_articlesets.filter(
Q(project=p.id) | Q(projects_set=p.id)).values_list("id",
flat=True))
no_favourites = not favs
favourites = json.dumps(list(favs | initial))
codingjobs = json.dumps(
tuple(
CodingJob.objects.filter(
articleset__in=all_articlesets).values_list(
"articleset_id", flat=True)))
all_sets = json.dumps(
tuple(all_articlesets.values_list("id", flat=True)))
form = SelectionForm(project=p,
data=self.request.GET,
initial={"datetype": "all"})
context.update(locals())
return context
def get_context_data(self, **kwargs):
query_id = self.request.GET.get("query", "null")
user_id = self.request.user.id
article_ids = self._get_ids("articles")
article_ids_lines = "\n".join(str(id) for id in article_ids)
articleset_ids = self._get_ids("sets")
codingjob_ids = self._get_ids("jobs")
codingjob_ids_json = json.dumps(list(codingjob_ids))
if codingjob_ids:
all_articlesets = self.project.favourite_articlesets.all().only(
"id", "name")
all_articlesets = all_articlesets.filter(
codingjob_set__id__in=codingjob_ids)
articleset_ids = all_articlesets.values_list("id", flat=True)
all_codingjobs = self.project.codingjob_set.all().filter(
id__in=codingjob_ids)
else:
all_codingjobs = None
all_articlesets = self.project.favourite_articlesets.all().only(
"id", "name")
all_articlesets = all_articlesets.filter(
codingjob_set__id__isnull=True)
articlesets = self.project.all_articlesets().filter(
id__in=articleset_ids).only("id", "name")
articlesets_names = (aset.name for aset in articlesets)
articleset_ids_json = json.dumps(list(articleset_ids))
codebooks = self.project.get_codebooks().order_by("name").only(
"id", "name")
saved_queries_table = self.get_saved_queries_table()
articlesets_table = self.get_articlesets_table()
codingjobs_table = self.get_codingjobs_table()
form = SelectionForm(project=self.project,
articlesets=articlesets,
codingjobs=all_codingjobs,
data=self.request.GET,
initial={
"datetype": "all",
"articlesets": articlesets
})
statistics = amcates.ES().statistics(
filters={"sets": list(articleset_ids)})
settings = conf.settings
saved_queries = Query.objects.filter(project=self.project)
if self.request.user.is_anonymous():
saved_user_queries, saved_project_queries = [], saved_queries
else:
saved_user_queries = saved_queries.filter(
user=self.request.user)[:SHOW_N_RECENT_QUERIES]
saved_project_queries = saved_queries.filter(~Q(
user=self.request.user))[:SHOW_N_RECENT_QUERIES]
form.fields["articlesets"].widget.attrs['disabled'] = 'disabled'
r_queryactions = sorted(q.__name__[:-6] for q in get_r_queryactions())
return dict(super(QueryView, self).get_context_data(), **locals())