def run(self, form):
assert isinstance(self.data, QueryDict), "Class should have been instantiated with a django QueryDict as 'data'"
selection = SelectionSearch.get_instance(form)
data = {API_KEYWORD_MAP.get(k, k): v for k, v in self.data.lists()}
data["q"] = ["{}#{}".format(q.label, q.query) for q in selection.get_queries()]
data["ids"] = data.get("ids", selection.get_filters().get("ids", []))
url = urlencode(data, doseq=True)
rowlink = ARTICLE_ROWLINK.format(reverse("navigator:project-details", args=[self.project.id]), "{id}")
table = Datatable(
SearchResource,
url="/api/v4/search",
rowlink=rowlink,
rowlink_open_in="new",
checkboxes=True,
allow_export_via_post=True,
allow_html_export=True
)
table = table.add_arguments(minimal="1")
table = table.add_arguments(project=str(self.project.id))
for k, vs in data.items():
for v in vs:
table = table.add_arguments(**{k:v})
return TABLE_TEMPLATE.render({"form": form, "url": url, "table": table})
def get_context_data(self, **kwargs):
ctx = super(CodingSchemaListView, self).get_context_data(**kwargs)
schemas = Datatable(CodingSchemaViewSet, rowlink="./{id}", url_kwargs=dict(project=self.project.id)).hide("highlighters")
owned_schemas = schemas.filter(project=self.project)
linked_schemas = schemas.filter(projects_set=self.project)
ctx.update(locals())
return ctx
def get_context_data(self, **kwargs):
ctx = super(CodingSchemaListView, self).get_context_data(**kwargs)
schemas = Datatable(CodingSchemaViewSet, rowlink="./{id}", url_kwargs=dict(project=self.project.id)).hide("highlighters")
owned_schemas = schemas.filter(project=self.project)
linked_schemas = schemas.filter(projects_set=self.project)
ctx.update(locals())
return ctx
def run(self, form):
assert isinstance(
self.data, QueryDict
), "Class should have been instantiated with a django QueryDict as 'data'"
selection = SelectionSearch(form)
data = {API_KEYWORD_MAP.get(k, k): v for k, v in self.data.lists()}
data["q"] = [
"{}#{}".format(q.label, q.query) for q in selection.get_queries()
]
data["ids"] = data.get("ids", selection.get_filters().get("ids", []))
url = urlencode(data, doseq=True)
rowlink = ARTICLE_ROWLINK.format(
reverse("navigator:project-details", args=[self.project.id]),
"{id}")
table = Datatable(SearchResource,
url="/api/v4/search",
rowlink=rowlink,
rowlink_open_in="new",
checkboxes=True,
allow_export_via_post=True,
allow_html_export=True)
table = table.add_arguments(minimal="1")
table = table.add_arguments(project=str(self.project.id))
for k, vs in data.items():
for v in vs:
table = table.add_arguments(**{k: v})
return TABLE_TEMPLATE.render(
Context({
"form": form,
"url": url,
"table": table
}))
def get_saved_queries_table(self):
table = Datatable(
QueryViewSet,
url_kwargs={"project": self.project.id},
rowlink="{id}"
)
table = table.hide("last_saved", "parameters", "project")
return table
def get_context_data(self, **kwargs):
ctx = super(CodebookListView, self).get_context_data(**kwargs)
all_codebooks = Datatable(CodebookViewSet, rowlink='./{id}', url_kwargs={"project" : self.project.id}).hide("codes")
owned_codebooks = all_codebooks.filter(project=self.project).hide("project")
linked_codebooks = all_codebooks.filter(projects_set=self.project)
ctx.update(locals())
return ctx
def test_search(self):
# Resources are not searchable (yet?)
d = Datatable(ProjectResource)
self.assertIn('"searching": false', str(d))
# Articleset viewsets are searchable
d = Datatable(ArticleSetViewSet, url_kwargs={"project": 1})
self.assertIn('"searching": true', str(d))
def get_context_data(self, **kwargs):
ctx = super(CodebookListView, self).get_context_data(**kwargs)
all_codebooks = Datatable(CodebookViewSet, rowlink='./{id}', url_kwargs={"project" : self.project.id})
owned_codebooks = all_codebooks.filter(project=self.project)
linked_codebooks = all_codebooks.filter(projects_set=self.project)
ctx.update(locals())
return ctx
def get_articlesets_table(self):
table = Datatable(FavouriteArticleSetViewSet,
url_kwargs={"project": self.project.id},
rowlink="?sets={id}",
checkboxes=True)
table = table.hide("favourite", "featured", "project", "provenance")
return table
def get_saved_queries_table(self):
table = Datatable(
QueryViewSet,
url_kwargs={"project": self.project.id},
rowlink=self.get_query_url("{id}")
)
table = table.filter(archived=False)
table = table.hide("archived", "last_saved", "parameters", "project")
return table
def get_articlesets_table(self):
table = Datatable(
FavouriteArticleSetViewSet,
url_kwargs={"project": self.project.id},
rowlink="?sets={id}",
checkboxes=True
)
table = table.hide("favourite", "featured", "project", "provenance")
return table
def test_hide(self):
d = Datatable(ProjectResource)
# Nothing hidden by default
self.assertEqual(set(d.fields), TestDatatable.PROJECT_FIELDS)
# Hide some fields..
hide = {"id", "name", "insert_user"}
d = d.hide(*hide)
self.assertEqual(set(d.fields), TestDatatable.PROJECT_FIELDS - hide)
def test_hide(self):
d = Datatable(ProjectResource)
# Nothing hidden by default
self.assertEqual(set(d.fields), TestDatatable.PROJECT_FIELDS)
# Hide some fields..
hide = {"id", "name", "insert_user"}
d = d.hide(*hide)
self.assertEqual(set(d.fields), TestDatatable.PROJECT_FIELDS - hide)
def getDatatable(form, rowlink='article/{id}'):
from api.rest.datatable import Datatable
from api.rest.resources import SearchResource
table = Datatable(SearchResource, rowlink=rowlink)
for field, val in filters_from_form(form):
table = table.filter(**{field : val})
for query in queries_from_form(form):
table = table.add_arguments(q=query.query)
if form.get('include_all') and form.get('include_all') != 'False':
table = table.add_arguments(q="*")
return table
def getDatatable(form, rowlink='article/{id}', **kwargs):
from api.rest.datatable import Datatable
from api.rest.resources import SearchResource
table = Datatable(SearchResource, rowlink=rowlink, **kwargs)
for field, val in filters_from_form(form):
table = table.filter(**{field : val})
for query in queries_from_form(form):
table = table.add_arguments(q=query.query)
if form.get('include_all') and form.get('include_all') != 'False':
table = table.add_arguments(q="*")
return table
def run(self, form):
selection = SelectionSearch(form)
data = {API_KEYWORD_MAP.get(k, k): v for k,v in self.data.iterlists()}
data["q"] = ["{}#{}".format(q.label, q.query) for q in selection.get_queries()]
url = urllib.urlencode(data, doseq=True)
table = Datatable(SearchResource, url="/api/v4/search")
table = table.add_arguments(minimal="1")
table = table.add_arguments(project=str(self.project.id))
for k, vs in data.items():
for v in vs:
table = table.add_arguments(**{k:v})
return TABLE_TEMPLATE.render(Context({"form": form, "url": url, "table": table}))
def view(request, id=None, form=None):
if id is None:
return redirect(reverse("navigator:user", args=[request.user.id]))
try:
user = User.objects.get(id=id)
except User.DoesNotExist:
return HttpResponse("User not found", status=404)
ref = request.META.get('HTTP_REFERER', '')
success = ref.endswith(reverse("navigator:user", args=[user.id
])) and not form
form = form or forms.UserDetailsForm(request, instance=user)
# Generate projects-table javascript
projects = Datatable(ProjectResource).filter(projectrole__user=user)
main_active = "Current User" if user == request.user else "Users"
return render(
request, "user_view.html", {
'user': user,
'form': form,
'projects': projects,
'success': success,
'main_active': main_active
})
def test_fields(self):
d = Datatable(ProjectResource)
self.assertEqual(set(d.fields), TestDatatable.PROJECT_FIELDS)
# Test order of fields.
class TestSerializer(AmCATModelSerializer):
class Meta:
model = Project
fields = ('name', 'description', 'id')
class TestResource(AmCATResource):
model = Project
serializer_class = TestSerializer
d = Datatable(TestResource)
self.assertEqual(('name', 'description', 'id'), tuple(d.fields))
def index(request):
"""Show unfinished jobs"""
jobs = Datatable(CodingJobResource)
jobs = jobs.rowlink_reverse("annotator:annotator-codingjob",
args=[9999999999])
jobs = jobs.filter(coder=request.user).hide(
'coder', 'articleset', 'unitschema',
'articleschema') #.filter(status='unfinished')
ctx = locals()
ctx.update({
'context': request.user,
'selected': 'unfinished jobs',
})
return render(request, 'codingjobs.html', locals())
def test_viewset(self):
"""Can ViewSets also be used?"""
dt = Datatable(CodingSchemaFieldViewSet, url_kwargs={"project": 1})
self.assertTrue(dt.url.startswith("/api/v4/projects/1/codingschemafields/"))
self.assertEqual(set(dt.fields), {
'id', 'codingschema', 'fieldnr', 'label', 'required', 'fieldtype',
'codebook', 'split_codebook', 'default',
})
def index(request, coder=None):
"""
Show unfinished jobs
"""
coder = coder if coder is not None else request.user
jobs = Datatable(CodingJobResource, rowlink='/annotator/codingjob/{id}')
jobs = jobs.filter(coder=coder).hide('coder',)#.filter(status='unfinished')
ctx = locals()
ctx.update({
'menu' : CODINGJOB_MENU,
'context' : coder,
'selected' : 'unfinished jobs'
})
return render(request, 'navigator/report/codingjobs.html', locals())
def index(request, coder=None):
"""
Show unfinished jobs
"""
is_firefox = "Firefox" in request.META["HTTP_USER_AGENT"]
coder = coder if coder is not None else request.user
jobs = Datatable(CodingJobResource, rowlink="/annotator/codingjob/{id}")
jobs = jobs.filter(coder=coder).hide(
"coder", "articleset", "unitschema", "articleschema"
) # .filter(status='unfinished')
ctx = locals()
ctx.update({"menu": CODINGJOB_MENU, "context": coder, "selected": "unfinished jobs"})
return render(request, "codingjobs.html", locals())
def index(request, coder=None):
"""
Show unfinished jobs
"""
coder = coder if coder is not None else request.user
jobs = Datatable(CodingJobResource, rowlink='/annotator/codingjob/{id}')
jobs = jobs.filter(coder=coder).hide('coder',
) #.filter(status='unfinished')
ctx = locals()
ctx.update({
'menu': CODINGJOB_MENU,
'context': coder,
'selected': 'unfinished jobs'
})
return render(request, 'navigator/report/codingjobs.html', locals())
def get_codingjobs_table(self):
table = Datatable(CodingJobViewSet,
url_kwargs={
"project": self.project.id
},
rowlink="?jobs={id}",
checkboxes=True).filter(archived=False).hide(
"articleset", "archived", "insertuser")
return table
def codebooks(request, project):
"""
Codebooks-tab.
"""
owned_codebooks = Datatable(
CodebookResource, rowlink='./codebook/{id}').filter(project=project)
linked_codebooks = (Datatable(
CodebookResource,
rowlink='./codebook/{id}').filter(projects_set=project))
can_import = True #project.can_update(request.user)
can_create = True #Codebook.can_create(request.user) and project.can_update(request.user)
deleted = session_pop(request.session, "deleted_codebook")
context = project
menu = PROJECT_MENU
selected = "codebooks"
return render(request, "navigator/project/codebooks.html", locals())
def index(request, coder=None):
"""
Show unfinished jobs
"""
is_firefox = "Firefox" in request.META["HTTP_USER_AGENT"]
coder = coder if coder is not None else request.user
jobs = Datatable(CodingJobResource, rowlink='/annotator/codingjob/{id}')
jobs = jobs.filter(coder=coder).hide('coder', 'articleset', 'unitschema', 'articleschema')#.filter(status='unfinished')
ctx = locals()
ctx.update({
'menu' : CODINGJOB_MENU,
'context' : coder,
'selected' : 'unfinished jobs'
})
return render(request, 'codingjobs.html', locals())
def schemas(request, project):
"""
Codingschemas-tab
"""
owned_schemas = Datatable(CodingSchemaResource,
rowlink='./schema/{id}').filter(project=project)
linked_schemas = (Datatable(
CodingSchemaResource,
rowlink='./schema/{id}').filter(projects_set=project))
ctx = {
'owned_schemas': owned_schemas,
'linked_schemas': linked_schemas,
'menu': PROJECT_MENU,
'selected': 'codingschemas',
'context': project,
'deleted': session_pop(request.session, "deleted_schema"),
}
return render(request, "navigator/project/schemas.html", ctx)
def scrape_articles(request, project):
scripts = (Datatable(ScraperResource,
rowlink="./scrape-articles/{id}").hide(
'module', 'class_name', 'username', 'password',
'run_daily', 'articleset'))
return table_view(request,
project,
scripts,
selected='article sets',
template='navigator/project/scrape.html')
def index(request, coder=None):
"""
Show unfinished jobs
"""
is_firefox = "Firefox" in request.META["HTTP_USER_AGENT"]
coder = coder if coder is not None else request.user
jobs = Datatable(CodingJobResource, rowlink='/annotator/codingjob/{id}')
jobs = jobs.filter(coder=coder).hide(
'coder', 'articleset', 'unitschema',
'articleschema') #.filter(status='unfinished')
ctx = locals()
ctx.update({
'menu': CODINGJOB_MENU,
'context': coder,
'selected': 'unfinished jobs'
})
return render(request, 'codingjobs.html', locals())
def run(self, form):
selection = SelectionSearch(form)
data = {API_KEYWORD_MAP.get(k, k): v for k, v in self.data.iterlists()}
data["q"] = [
"{}#{}".format(q.label, q.query) for q in selection.get_queries()
]
url = urllib.urlencode(data, doseq=True)
table = Datatable(SearchResource, url="/api/v4/search")
table = table.add_arguments(minimal="1")
table = table.add_arguments(project=str(self.project.id))
for k, vs in data.items():
for v in vs:
table = table.add_arguments(**{k: v})
return TABLE_TEMPLATE.render(
Context({
"form": form,
"url": url,
"table": table
}))
def test_filter(self):
d = Datatable(UserResource)
s = '/api/v4/user?format=json'
# No filter
self.assertEqual(d.url, s)
# One filter
d = d.filter(id=1)
self.assertEqual(d.url, s + "&id=1")
# Multiple filters
d = d.filter(id=2)
self.assertEqual(d.url, s + "&id=1&id=2")
d = Datatable(UserResource).filter(id=[1, 2])
self.assertEqual(d.url, s + "&id=1&id=2")
# Test can allow illegal filter field as extra_arg
d = Datatable(UserResource).add_arguments(q=[1, 2])
self.assertEqual(d.url, s + "&q=1&q=2")
def test_order_by(self):
d = Datatable(ProjectResource).order_by("name")
self.assertTrue("name" in str(d))
self.assertTrue('["name", "asc"]' in str(d))
self.assertTrue('["name", "desc"]' in str(d.order_by("-name")))
with self.assertRaises(ValueError):
d.order_by("bla")
with self.assertRaises(ValueError):
d.order_by("?name")
def all(request, coder):
"""
Show all (including finished) jobs
"""
jobs = Datatable(CodingJobResource).filter(coder=coder)
ctx = locals()
ctx.update({
'jobs': jobs,
'menu': CODINGJOB_MENU,
'context': coder,
'selected': 'all jobs'
})
return render(request, 'navigator/report/codingjobs.html', ctx)
def schema(request, schema, project):
fields = (Datatable(CodingSchemaFieldResource).filter(
codingschema=schema).hide('codingschema'))
return table_view(
request,
project,
fields,
'codingschemas',
template="navigator/project/schema.html",
schema=schema,
is_new=session_pop(request.session,
"schema_{}_is_new".format(schema.id), False),
is_edited=session_pop(request.session,
"schema_{}_edited".format(schema.id), False))
def users_view(request, project):
"""
View all users affiliated with this project. Also render a form
to add users to the project (if permissions are met).
"""
users = Datatable(ProjectRoleResource, rowlink='./user/{user_id}')\
.filter(project=project).hide('project', 'id')
if request.user.get_profile().haspriv('manage_project_users', project):
add_user = forms.ProjectRoleForm(project)
ctx = dict(locals())
ctx.update({'menu': PROJECT_MENU, 'selected': 'users', 'context': project})
return render(request, 'navigator/project/users.html', ctx)
def test_order_by(self):
d = Datatable(ProjectResource).order_by("name")
self.assertTrue("name" in str(d))
self.assertTrue('["name", "asc"]' in str(d))
self.assertTrue('["name", "desc"]' in str(d.order_by("-name")))
with self.assertRaises(ValueError):
d.order_by("bla")
with self.assertRaises(ValueError):
d.order_by("?name")
def view_codingjob(request, codingjob, project):
"""
View and edit a codingjob
"""
form = forms.CodingJobForm(data=(request.POST or None), instance=codingjob)
articles = Datatable(ArticleMetaResource)\
.filter(articleset=codingjob.articleset.id)\
.hide("section", "pagenr", "byline", "metastring", "url")\
.hide("project", "medium", "text", "uuid")
if form.is_valid() and form.save():
return redirect(reverse(codingjobs, args=[project.id]))
ctx = locals()
ctx.update(dict(menu=PROJECT_MENU, context=project))
return render(request, 'navigator/project/edit_codingjob.html', ctx)
def analysedsentence(request, project, sentence, rulesetid=None):
tokens = (sentence.tokens.all().select_related(
"word", "word__word",
"word__lemma").prefetch_related("triples", "triples__child",
"triples__parent",
"triples__relation"))
soh = SOHServer(url="http://localhost:3030/x")
tree = syntaxtree.SyntaxTree(soh, tokens)
parsetree = tree.visualise().getHTMLObject()
menu = PROJECT_MENU
context = project
rulesets = Datatable(RuleSet).rowlink_reverse(
"analysedsentence-ruleset",
args=[project.id, sentence.id,
"{id}"]) #, rowlink="./upload-articles/{id}")
if rulesetid:
ruleset = RuleSet.objects.get(pk=rulesetid)
trees = []
ruleset_error = None
tree.apply_lexicon(ruleset.lexicon_codebook, ruleset.lexicon_language)
parsetree = tree.visualise().getHTMLObject()
grey_rel = lambda triple: ({
'color': 'grey'
} if 'rel_' in triple.predicate else {})
for rule in ruleset.rules.all():
try:
tree.apply_rule(rule)
if rule.display:
trees.append(
(rule, tree.visualise(
triple_args_function=grey_rel).getHTMLObject()))
except Exception, e:
ruleset_error = "Exception processing rule {rule.order}: {rule.label}\n\n{e}".format(
**locals())
break
if not ruleset_error:
finaltree = tree.visualise(
triple_args_function=grey_rel).getHTMLObject()
def codingjobs(request, project):
"""
Coding-jobs tab
"""
cdjobs = (Datatable(CodingJobResource, rowlink='./codingjob/{id}').filter(
project=project).hide('project').order_by("-insertdate"))
deleted = session_pop(request.session, "deleted_codingjob")
added = session_pop(request.session, "added_codingjob")
if added:
added = [CodingJob.objects.get(pk=i) for i in added]
return table_view(request,
project,
cdjobs,
'codingjobs',
template="navigator/project/codingjobs.html",
added=added,
deleted=deleted)
def upload_article(request, project):
plugin_type = UploadScript.get_plugin_type()
#scripts = (Datatable(PluginResource, rowlink="./upload-articles/{id}")
# .filter(plugin_type=plugin_type)
# .hide('id', 'class_name', 'plugin_type'))
scripts = (Datatable(
PluginResource,
rowlink="./upload-articles/{id}").filter(plugin_type=plugin_type).hide(
'id', 'plugin_type'))
can_create_plugin = False #Plugin.can_create(request.user)
return table_view(request,
project,
scripts,
selected='article sets',
template='navigator/project/upload.html',
can_create_plugin=can_create_plugin,
plugin_type=plugin_type)
def preprocessing(request, project):
"""
Codebooks-tab.
"""
table = Datatable(AnalysedArticleResource).filter(
article__articlesets_set__project=project)
form = AssignParsing.options_form(request.POST or None)
form.fields['articleset'].queryset = ArticleSet.objects.filter(
pk__in=project.all_articlesets())
if form.is_valid():
assigned_n = AssignParsing(form).run()
assigned_plugin = form.cleaned_data["plugin"]
assigned_set = form.cleaned_data["articleset"]
context = project
menu = PROJECT_MENU
selected = "preprocessing"
return render(request, "navigator/project/preprocessing.html", locals())
def test_filter(self):
d = Datatable(UserResource)
s = '/api/v4/user?format=json'
# No filter
self.assertEqual(d.url, s)
# One filter
d = d.filter(id=1)
self.assertEqual(d.url, s + "&id=1")
# Multiple filters
d = d.filter(id=2)
self.assertEqual(d.url, s + "&id=1&id=2")
d = Datatable(UserResource).filter(id=[1, 2])
self.assertEqual(d.url, s + "&id=1&id=2")
# Test can allow illegal filter field as extra_arg
d = Datatable(UserResource).add_arguments(q=[1, 2])
self.assertEqual(d.url, s + "&q=1&q=2")
def view(request, user=None, form=None):
if user is None:
return redirect(reverse(view, args=[request.user.id]))
ref = request.META.get('HTTP_REFERER', '')
success = ref.endswith(reverse(view, args=[user.id])) and not form
form = form or forms.UserDetailsForm(request, instance=user)
# Generate projects-table javascript
projects = Datatable(ProjectResource).filter(projectrole__user=user)
menu = None if user == request.user else USER_MENU
return render(
request, "navigator/user/view.html", {
'user': user,
'form': form,
'projects': projects,
'success': success,
'menu': menu
})
def test_get_name(self):
d = Datatable(ProjectResource).filter(id=[1, "#$^"])
self.assertTrue(len(d.get_name()) >= 1)
self.assertFalse(re.match(r'[^0-9A-Za-z_:.-]', d.get_name()))
self.assertTrue(re.match(r'^[A-Za-z]', d.get_name()))
def test_js(self):
d = Datatable(ProjectResource)
js = d.get_js()
def get_datatable(self, **kwargs):
"""Create the Datatable object"""
table = Datatable(self.get_resource(), rowlink=self.get_rowlink(), **kwargs)
table = self.filter_table(table)
return table
def index(request):
all_schemas = Datatable(CodingSchemaResource, options=dict(iDisplayLength=10))
own_schemas = all_schemas.filter(project__in=request.user.projects)
return render(request, 'navigator/schemas/index.html', locals())
