def run(self): sf = SelectionForm(self.project, self.data) sf.full_clean() article_ids = list(keywordsearch.get_ids(sf.cleaned_data)) result = SaveAsSetScript(self.data).run(article_ids) result.provenance = json.dumps(dict(self.data)) result.save() return self.outputResponse(result, object)
def run(self): sel = SelectionForm(project=self.project, data=self.data) if not sel.is_valid(): # This should not happen when using normal pages (i.e., someone is trying # to hack) forms.ValidationError("Non-valid values entered.") sel.full_clean() if sel.use_solr: sel.cleaned_data['length'] = 99999999 # Return 'unlimited' results sel.cleaned_data['sortColumn'] = None sel.cleaned_data['start'] = 0 sel.cleaned_data['columns'] = [] articles = solrlib.getArticles(sel.cleaned_data) else: articles = database.get_queryset(**sel.cleaned_data) project = Project.objects.get(id=self.data.get('projects')) # Create articleset a = ArticleSet.objects.create(project=project, name=self.data['setname']) a.add(*articles) # Split all articles CreateSentences(dict(articlesets=[a.id])).run() # Create codingjob coder = User.objects.get(id=self.data['coder']) articleschema = CodingSchema.objects.get(id=self.data['articleschema']) unitschema = CodingSchema.objects.get(id=self.data['unitschema']) if not 'insertuser' in self.data: insertuser = auth.get_request().user else: insertuser = User.objects.get(id=self.data['insertuser']) c = CodingJob.objects.create(project=project, name=self.data['setname'], articleset=a, coder=coder, articleschema=articleschema, unitschema=unitschema, insertuser=insertuser) html = "<div>Saved as <a href='%s'>coding job %s</a>.</div>" return HttpResponse(json.dumps({ "html": html % (reverse("codingjob", args=[project.id, c.id]), c.id), "webscriptClassname": self.__class__.__name__, "webscriptName": self.name, "doNotAddActionToMainForm": True }), mimetype='application/json')
def run(self): self.progress_monitor.update(1, "Listing articles") sf = SelectionForm(self.project, self.data) sf.full_clean() article_ids = list(keywordsearch.get_ids(sf.cleaned_data)) self.progress_monitor.update(39, "Creating set with {n} articles".format(n=len(article_ids))) result = SaveAsSetScript(self.data, monitor=self.progress_monitor).run(article_ids) result.provenance = json.dumps(dict(self.data)) result.save() return self.outputResponse(result, object)
def run(self): self.progress_monitor.update(1, "Listing articles") sf = SelectionForm(self.project, self.data) sf.full_clean() article_ids = list(keywordsearch.get_ids(sf.cleaned_data)) self.progress_monitor.update( 39, "Creating set with {n} articles".format(n=len(article_ids))) result = SaveAsSetScript( self.data, monitor=self.progress_monitor).run(article_ids) result.provenance = json.dumps(dict(self.data)) result.save() return self.outputResponse(result, object)
def run(self): selection = SelectionForm(project=self.project, data=self.data) selection.full_clean() aggrTable = AggregationScript(project=self.project, options=self.data).run() if self.output == 'json-html' or (self.output == 'html' and self.options['graphOnly'] == True): columns = sorted(aggrTable.getColumns(), key=lambda x: x.id if hasattr(x, 'id') else x) dataTablesHeaderDict = [{'mDataProp':TITLE_COLUMN_NAME,'sTitle':TITLE_COLUMN_NAME, 'sType':'objid', 'sWidth':'100px'}] + \ [{'mDataProp':get_key(col),'mData':get_key(col),'sTitle':col, 'sWidth':'70px'} for col in columns] dataJson = [] for row in aggrTable.getRows(): rowJson = dict([(get_key(col), aggrTable.getValue(row, col)) for col in columns]) rowJson[TITLE_COLUMN_NAME] = row dataJson.append(rowJson) if self.options['xAxis'] == 'date': datesDict = dict( _getDatesDict(aggrTable, self.options['dateInterval'])) else: datesDict = {} labels = {q.label: q.query for q in aggrTable.queries} aggregationType = 'hits' if self.options[ 'counterType'] == 'numberOfHits' else 'articles' graphOnly = 'true' if self.options['graphOnly'] == True else 'false' scriptoutput = render_to_string( 'api/webscripts/aggregation.html', { 'dataJson': json.dumps(dataJson), 'columnsJson': json.dumps(dataTablesHeaderDict), 'aggregationType': aggregationType, 'datesDict': json.dumps(datesDict), 'graphOnly': graphOnly, 'labels': json.dumps(labels), 'ownForm': self.form(project=self.project, data=self.data), 'relative': int(self.options['relative']) }) if self.output == 'json-html': return self.outputJsonHtml(scriptoutput) else: return HttpResponse(scriptoutput, mimetype='text/html') else: # json-html output return self.outputResponse(aggrTable, AggregationScript.output_type)
def run(self): sel = SelectionForm(project=self.project, data=self.data) if not sel.is_valid(): # This should not happen when using normal pages (i.e., someone is trying # to hack) forms.ValidationError("Non-valid values entered.") articles = list(keywordsearch.get_ids(self.data)) # Create articleset a = ArticleSet.create_set(project=self.project, articles=articles, name=self.data['setname'], favourite=False) # Split all articles CreateSentences(dict(articlesets=[a.id])).run() # Create codingjob coder = User.objects.get(id=self.data['coder']) articleschema = CodingSchema.objects.get(id=self.data['articleschema']) unitschema = CodingSchema.objects.get(id=self.data['unitschema']) if not 'insertuser' in self.data: insertuser = auth.get_request().user else: insertuser = User.objects.get(id=self.data['insertuser']) c = CodingJob.objects.create(project=self.project, name=self.data['setname'], articleset=a, coder=coder, articleschema=articleschema, unitschema=unitschema, insertuser=insertuser) html = "<div>Saved as <a href='%s'>coding job %s</a>.</div>" return HttpResponse(json.dumps({ "html": html % (reverse("coding job-details", args=[self.project.id, c.id ]), c.id), "webscriptClassname": self.__class__.__name__, "webscriptName": self.name, "doNotAddActionToMainForm": True }), mimetype='application/json')
def outputJsonHtml(self, scriptoutput): actions = self.getActions() webscriptform = None if self.form: try: webscriptform = self.form(project=self.project, data=self.data) except TypeError: webscriptform = self.form(data=self.data) selectionform = SelectionForm(project=self.project, data=self.data) html = render_to_string('navigator/selection/webscriptoutput.html', { 'scriptoutput':scriptoutput, 'actions': actions, 'selectionform':selectionform, 'webscriptform':webscriptform }) reply = {} reply['html'] = html reply['queries'] = getQueries() reply['querytime'] = '%.2f' % (time.time() - self.initTime) reply['webscriptName'] = self.name reply['webscriptClassname'] = self.__class__.__name__ jsondata = json.dumps(reply, default=lambda o:unicode(o)) return HttpResponse(jsondata, mimetype='text/plain')
def _run_query(self, form_data, expected_indices=None, expected_count=None, msg=None): self._setUp() sets = ArticleSet.objects.filter(pk=self.articleset.pk) form = SelectionForm(articlesets=sets, project=self.articleset.project, data=form_data) form.full_clean() self.assertFalse(form.errors, "Form contains errors") search = SelectionSearch(form) if expected_indices: article_ids = search.get_article_ids() articles = Article.objects.filter(id__in=article_ids) expected = [self.articles[i] for i in expected_indices] self.assertSetEqual(set(articles), set(expected), msg=msg) if expected_count: self.assertEqual(search.get_count(), expected_count, msg=msg)
def index(request): """returns the selection form""" outputs = [] for ws in mainScripts: outputs.append({'id':ws.__name__, 'name':ws.name, 'formAsHtml': ws.formHtml()}) context = {'form':SelectionForm(data=request.REQUEST), 'outputs':outputs} return render(request, 'navigator/selection/form.html', context)
def run(self): selection = SelectionForm(project=self.project, data=self.data) selection.full_clean() aggrTable = AggregationScript(project=self.project, options=self.data, monitor=self.progress_monitor).run() if self.output == 'json-html' or (self.output == 'html' and self.options['graphOnly'] == True): columns = sorted(aggrTable.getColumns(), key=lambda x:x.id if hasattr(x,'id') else x) dataTablesHeaderDict = [{'mDataProp':TITLE_COLUMN_NAME,'sTitle':TITLE_COLUMN_NAME, 'sType':'objid', 'sWidth':'100px'}] + \ [{'mDataProp':get_key(col),'mData':get_key(col),'sTitle':col, 'sWidth':'70px'} for col in columns] dataJson = [] for row in aggrTable.getRows(): rowJson = dict([(get_key(col), aggrTable.getValue(row, col)) for col in columns]) rowJson[TITLE_COLUMN_NAME] = row dataJson.append(rowJson) if self.options['xAxis'] == 'date': datesDict = dict(_getDatesDict(aggrTable, self.options['dateInterval'])) else: datesDict = {} labels = {q.label : q.query for q in aggrTable.queries} aggregationType = 'articles' graphOnly = 'true' if self.options['graphOnly'] == True else 'false' scriptoutput = render_to_string('api/webscripts/aggregation.html', { 'dataJson':json.dumps(dataJson), 'columnsJson':json.dumps(dataTablesHeaderDict), 'aggregationType':aggregationType, 'datesDict': json.dumps(datesDict), 'graphOnly': graphOnly, 'labels' : json.dumps(labels), 'ownForm':self.form(project=self.project, data=self.data), 'relative':int(self.options['relative']) }) if self.output == 'json-html': return self.outputJsonHtml(scriptoutput) else: return HttpResponse(scriptoutput, mimetype='text/html') else: # json-html output return self.outputResponse(aggrTable, AggregationScript.output_type)
def run(self): formData = self.data.copy() # copy needed since formData is inmutable if "articlesets" not in formData: artsets = [str(aset.id) for aset in Project.objects.get(id=formData['projects']).all_articlesets()] formData.setlist("articlesets", artsets) if isinstance(self.data['projects'], (basestring, int)): project_id = int(self.data['projects']) else: project_id = int(self.data['projects'][0]) formData["start_date"] = formData["start_date"].split("T")[0] formData["end_date"] = formData["end_date"].split("T")[0] sf = SelectionForm(self.project, formData) if not sf.is_valid(): raise ValueError(dict(sf._errors)) t = keywordsearch.getDatatable(sf.cleaned_data, rowlink_open_in="new", allow_html_export=True) t = t.rowlink_reverse("project-article-details", args=[project_id, '{id}']) cols = {FORM_FIELDS_TO_ELASTIC.get(f, f) for f in self.data.getlist('columns')} for f in list(t.get_fields()): if f not in cols: t = t.hide(f) if 'kwic' in cols and not self.data.get('query'): raise Exception("Cannot provide Keyword in Context without query") for col in cols & {'hits', 'text', 'lead', 'kwic'}: t = t.add_arguments(col=col) html = unicode(t) stats_log.info(json.dumps({ "actions": "query:articlelist", "user": self.user.username, "project_id": self.project.id, "project__name": self.project.name })) if self.output == "html": response = HttpResponse(mimetype='text/html') response.write(html) return response else: return self.outputJsonHtml(html)
def run(self): self.progress_monitor.update(1, "Creating summary") if isinstance(self.data['projects'], (basestring, int)): project_id = int(self.data['projects']) else: project_id = int(self.data['projects'][0]) sf = SelectionForm(self.project, self.data) sf.full_clean() n = keywordsearch.get_total_n(sf.cleaned_data) self.progress_monitor.update(39, "Found {n} articles in total".format(**locals())) articles = list(ArticleListScript(self.data).run()) for a in articles: a.hack_project_id = project_id self.output_template = 'api/webscripts/articlelist.html' self.progress_monitor.update(40, "Created summary") return self.outputResponse(dict(articlelist=articles, n=n, page=self.data.get('start')), ArticleListScript.output_type)
def run(self): sel = SelectionForm(project=self.project, data=self.data) if not sel.is_valid(): # This should not happen when using normal pages (i.e., someone is trying # to hack) forms.ValidationError("Non-valid values entered.") sel.full_clean() if sel.use_solr: sel.cleaned_data['length'] = 99999999 # Return 'unlimited' results sel.cleaned_data['sortColumn'] = None sel.cleaned_data['start'] = 0 sel.cleaned_data['columns'] = [] articles = solrlib.getArticles(sel.cleaned_data) else: articles = database.get_queryset(**sel.cleaned_data) project = Project.objects.get(id=self.data.get('projects')) # Create articleset a = ArticleSet.objects.create(project=project, name=self.data['setname']) a.add(*articles) # Split all articles CreateSentences(dict(articlesets=[a.id])).run() # Create codingjob coder = User.objects.get(id=self.data['coder']) articleschema = CodingSchema.objects.get(id=self.data['articleschema']) unitschema = CodingSchema.objects.get(id=self.data['unitschema']) if not 'insertuser' in self.data: insertuser = auth.get_request().user else: insertuser = User.objects.get(id=self.data['insertuser']) c = CodingJob.objects.create(project=project, name=self.data['setname'], articleset=a, coder=coder, articleschema=articleschema, unitschema=unitschema, insertuser=insertuser) html = "<div>Saved as <a href='%s'>coding job %s</a>.</div>" return HttpResponse(json.dumps({ "html" : html % (reverse("codingjob", args=[project.id, c.id]), c.id), "webscriptClassname" : self.__class__.__name__, "webscriptName" : self.name, "doNotAddActionToMainForm" : True }), mimetype='application/json')
def run(self): sel = SelectionForm(project=self.project, data=self.data) if not sel.is_valid(): # This should not happen when using normal pages (i.e., someone is trying # to hack) forms.ValidationError("Non-valid values entered.") articles = list(keywordsearch.get_ids(self.data)) # Create articleset a = ArticleSet.create_set(project=self.project, articles=articles, name=self.data['setname'], favourite=False) # Split all articles CreateSentences(dict(articlesets=[a.id])).run() # Create codingjob coder = User.objects.get(id=self.data['coder']) articleschema = CodingSchema.objects.get(id=self.data['articleschema']) unitschema = CodingSchema.objects.get(id=self.data['unitschema']) if not 'insertuser' in self.data: insertuser = auth.get_request().user else: insertuser = User.objects.get(id=self.data['insertuser']) c = CodingJob.objects.create(project=self.project, name=self.data['setname'], articleset=a, coder=coder, articleschema=articleschema, unitschema=unitschema, insertuser=insertuser) html = "<div>Saved as <a href='%s'>coding job %s</a>.</div>" return HttpResponse(json.dumps({ "html" : html % (reverse("coding job-details", args=[self.project.id, c.id]), c.id), "webscriptClassname" : self.__class__.__name__, "webscriptName" : self.name, "doNotAddActionToMainForm" : True }), mimetype='application/json')
def selection(request, project): """ Render article selection page. TODO: - update to meet PEP8 style - remove/replace webscripts (?) """ outputs = [] for ws in mainScripts: outputs.append({ 'id': ws.__name__, 'name': ws.name, 'formAsHtml': ws.formHtml(project=project) }) all_articlesets = project.all_articlesets() favs = tuple( project.favourite_articlesets.filter( Q(project=project.id) | Q(projects_set=project.id)).values_list( "id", flat=True)) no_favourites = not favs favourites = json.dumps(favs) codingjobs = json.dumps( tuple( CodingJob.objects.filter( articleset__in=all_articlesets).values_list("articleset_id", flat=True))) all_sets = json.dumps(tuple(all_articlesets.values_list("id", flat=True))) ctx = locals() ctx.update({ 'form': SelectionForm(project=project, data=request.GET, initial={"datetype": "all"}), 'outputs': outputs, 'project': project, 'context': project, 'menu': PROJECT_MENU, 'selected': 'query' }) return render(request, 'navigator/project/selection.html', ctx)
def get_context_data(self, **kwargs): articleset_ids = set( map( int, filter(unicode.isdigit, self.request.GET.get("sets", "").split(",")))) articlesets = self.project.all_articlesets().filter( id__in=articleset_ids) articlesets = articlesets.only("id", "name") query_id = self.request.GET.get("query", "null") user_id = self.request.user.id articleset_ids_json = json.dumps(list(articleset_ids)) codebooks = self.project.get_codebooks().order_by("name").only( "id", "name") all_articlesets = self.project.all_articlesets().filter( codingjob_set__id__isnull=True).only("id", "name") saved_queries_table = self.get_saved_queries_table() articlesets_table = self.get_articlesets_table() form = SelectionForm(project=self.project, articlesets=articlesets, data=self.request.GET, initial={ "datetype": "all", "articlesets": articlesets }) statistics = amcates.ES().statistics( filters={"sets": list(articleset_ids)}) settings = conf.settings saved_queries = Query.objects.filter(project=self.project) saved_user_queries = saved_queries.filter( user=self.request.user)[:SHOW_N_RECENT_QUERIES] saved_project_queries = saved_queries.filter(~Q( user=self.request.user))[:SHOW_N_RECENT_QUERIES] form.fields["articlesets"].widget.attrs['disabled'] = 'disabled' return dict(super(QueryView, self).get_context_data(), **locals())
def get_context_data(self, **kwargs): context = super(QueryView, self).get_context_data(**kwargs) p = self.project outputs = [ dict(id=ws.__name__, name=ws.name, formAsHtml=ws.formHtml(project=p)) for ws in mainScripts ] all_articlesets = p.all_articlesets() initial = { int(aset) for aset in self.request.GET.getlist("articlesets") if aset.isnumeric() } initial = initial.intersection( set(all_articlesets.values_list("id", flat=True))) favs = set( p.favourite_articlesets.filter( Q(project=p.id) | Q(projects_set=p.id)).values_list("id", flat=True)) no_favourites = not favs favourites = json.dumps(list(favs | initial)) codingjobs = json.dumps( tuple( CodingJob.objects.filter( articleset__in=all_articlesets).values_list( "articleset_id", flat=True))) all_sets = json.dumps( tuple(all_articlesets.values_list("id", flat=True))) form = SelectionForm(project=p, data=self.request.GET, initial={"datetype": "all"}) context.update(locals()) return context
def get_context_data(self, **kwargs): query_id = self.request.GET.get("query", "null") user_id = self.request.user.id article_ids = self._get_ids("articles") article_ids_lines = "\n".join(str(id) for id in article_ids) articleset_ids = self._get_ids("sets") codingjob_ids = self._get_ids("jobs") codingjob_ids_json = json.dumps(list(codingjob_ids)) if codingjob_ids: all_articlesets = self.project.favourite_articlesets.all().only( "id", "name") all_articlesets = all_articlesets.filter( codingjob_set__id__in=codingjob_ids) articleset_ids = all_articlesets.values_list("id", flat=True) all_codingjobs = self.project.codingjob_set.all().filter( id__in=codingjob_ids) else: all_codingjobs = None all_articlesets = self.project.favourite_articlesets.all().only( "id", "name") all_articlesets = all_articlesets.filter( codingjob_set__id__isnull=True) articlesets = self.project.all_articlesets().filter( id__in=articleset_ids).only("id", "name") articlesets_names = (aset.name for aset in articlesets) articleset_ids_json = json.dumps(list(articleset_ids)) codebooks = self.project.get_codebooks().order_by("name").only( "id", "name") saved_queries_table = self.get_saved_queries_table() articlesets_table = self.get_articlesets_table() codingjobs_table = self.get_codingjobs_table() form = SelectionForm(project=self.project, articlesets=articlesets, codingjobs=all_codingjobs, data=self.request.GET, initial={ "datetype": "all", "articlesets": articlesets }) statistics = amcates.ES().statistics( filters={"sets": list(articleset_ids)}) settings = conf.settings saved_queries = Query.objects.filter(project=self.project) if self.request.user.is_anonymous(): saved_user_queries, saved_project_queries = [], saved_queries else: saved_user_queries = saved_queries.filter( user=self.request.user)[:SHOW_N_RECENT_QUERIES] saved_project_queries = saved_queries.filter(~Q( user=self.request.user))[:SHOW_N_RECENT_QUERIES] form.fields["articlesets"].widget.attrs['disabled'] = 'disabled' r_queryactions = sorted(q.__name__[:-6] for q in get_r_queryactions()) return dict(super(QueryView, self).get_context_data(), **locals())