def search_view(request, mission, file_type):
""" Django view function to perform the collection search.
"""
collection = models.Collection.objects.get(mission=mission,
file_type=file_type)
available_search_fields = get_available_search_fields(collection)
results = None
if request.method == "POST":
search_form = forms.SearchForm(collection.locations.order_by("pk"),
available_search_fields, request.POST)
pagination_form = forms.PaginationForm(request.POST)
result_list_form = forms.RecordSearchResultListForm(
collection.locations.order_by("pk"), request.POST)
forms_valid = (search_form.is_valid() and pagination_form.is_valid()
and result_list_form.is_valid())
if forms_valid:
search_data = search_form.cleaned_data
location_ids = search_data.pop("locations", ())
# TODO: only scene centre??
footprint_or_scene_centre = search_data.pop(
"area_footprint_or_scene_centre", "footprint")
locations = collection.locations.order_by("pk")
if location_ids:
locations = locations.filter(id__in=location_ids)
observer = monitor("search_overview", **search_data)
with observer:
results = []
for location in locations:
qs = queries.search(
collection, search_data, location.records.all(),
footprint_or_scene_centre == "footprint")
values = qs.aggregate(volume=Sum("filesize"),
count=Count("filename"))
results.append((location, values))
else:
search_form = forms.SearchForm(collection.locations.order_by("pk"),
available_search_fields)
pagination_form = forms.PaginationForm(initial={
'page': '1',
'records_per_page': '15'
})
result_list_form = forms.RecordSearchResultListForm(
collection.locations.order_by("pk"))
return render(
request, "inventory/collection/search.html", {
"collections": models.Collection.objects.all(),
"search_form": search_form,
"pagination_form": pagination_form,
"result_list_form": result_list_form,
"collection": collection,
"results": results
})
def restart(self, job_id):
job = models.Job.objects.get(id=job_id)
kwargs = job.get_arguments()
with monitor(job.task, **kwargs):
return self._tasks[job.task](**kwargs)
def run(self, task, **kwargs):
name = task if isinstance(task, basestring) else task.task
with monitor(task, **kwargs):
return self._tasks[name](**kwargs)
def alignment_view(request, mission, file_type):
""" Django view function to perform the alignment check.
"""
collection = models.Collection.objects.get(mission=mission,
file_type=file_type)
config = collection.configuration
records = None
locations = None
locations_with_no_checksum = None
frmt = "html"
if request.method == "POST":
form = forms.AlignmentForm(collection.locations.order_by("pk"),
config.available_alignment_fields or [],
request.POST)
pagination_form = forms.PaginationForm(request.POST)
if form.is_valid() and pagination_form.is_valid():
frmt = form.cleaned_data.pop("format")
observer = monitor("alignment_check",
mission=mission,
file_type=file_type,
**form.cleaned_data)
with observer:
locations, qs = queries.alignment(collection,
form.cleaned_data)
page = pagination_form.cleaned_data.pop("page")
per_page = pagination_form.cleaned_data.pop("records_per_page")
if frmt == "html":
records = Paginator(qs, per_page).page(page)
else:
records = qs
locations_with_no_checksum = [
location for location in locations or []
if collection.get_metadata_field_mapping(location).get(
"checksum") is None
]
else:
form = forms.AlignmentForm(collection.locations.order_by("pk"),
config.available_alignment_fields or [])
pagination_form = forms.PaginationForm(initial={
'page': '1',
'records_per_page': '15'
})
if frmt == "html":
return render(
request, "inventory/collection/alignment.html", {
"collections": models.Collection.objects.all(),
"collection": collection,
"alignment_form": form,
"pagination_form": pagination_form,
"records": records,
"locations": locations,
"locations_with_no_checksum": locations_with_no_checksum
})
elif frmt in ("csv", "tsv"):
f = tempfile.SpooledTemporaryFile()
writer = csv.writer(f, delimiter="," if frmt == "csv" else "\t")
header = ["filename"]
for location in locations:
header.extend(
["checksum %s" % location.url,
"prevalence %s" % location.url])
header.append("annotations")
writer.writerow(header)
for row in qs:
writer.writerow([row["filename"]] + [
item for incidence in row["incidences"] for item in incidence
] + list(row["annotations"]))
size = f.tell()
f.seek(0)
response = StreamingHttpResponse(f, content_type="text/csv")
response["Content-Length"] = str(size)
response["Content-Disposition"] = (
'inline; filename="alignment-%s.csv"' %
(now().replace(microsecond=0, tzinfo=None).isoformat("T")))
return response
def result_list_view(request, mission, file_type):
"""
"""
collection = models.Collection.objects.get(mission=mission,
file_type=file_type)
location = None
available_search_fields = get_available_search_fields(collection)
config = collection.configuration
all_choices = dict((("checksum", "Checksum"), ) +
models.SEARCH_FIELD_CHOICES)
display_fields = SortedDict(
(field_id, all_choices[field_id])
for field_id in config.available_result_list_fields or [])
results = None
result_list = None
result_list_location = None
if request.method == "POST":
search_form = search_form = forms.SearchForm(
collection.locations.order_by("pk"), available_search_fields,
request.POST)
pagination_form = forms.PaginationForm(request.POST)
result_list_form = forms.RecordSearchResultListForm(
collection.locations.order_by("pk"), request.POST)
forms_valid = (search_form.is_valid() and pagination_form.is_valid()
and result_list_form.is_valid())
if forms_valid:
search_data = search_form.cleaned_data
result_list_location_id = result_list_form.cleaned_data[
"result_list_location"]
# TODO: only scene centre??
footprint_or_scene_centre = search_data.pop(
"area_footprint_or_scene_centre", "footprint")
location = collection.locations.get(id=result_list_location_id)
observer = monitor("search_results", **search_data)
with observer:
qs = queries.search(collection, search_data,
location.records.all(),
footprint_or_scene_centre == "footprint")
sort = result_list_form.cleaned_data.pop("sort", None)
if sort:
qs = qs.order_by(sort)
page = pagination_form.cleaned_data.pop("page")
per_page = pagination_form.cleaned_data.pop("records_per_page")
result_list = Paginator(qs, per_page).page(page)
result_list_location = location
# see if we need to create annotations
add_annotation_list_form = forms.AddAnnotationListForm(
request.POST)
if add_annotation_list_form.is_valid() and \
add_annotation_list_form.cleaned_data["text"] and \
add_annotation_list_form.cleaned_data["do_add_annotation"]:
ids = map(int, request.POST.getlist("add_record_annotation",
[]))
if add_annotation_list_form.cleaned_data[
"add_annotations_to_all"]:
annotation_qs = qs
else:
annotation_qs = location.records.filter(pk__in=ids)
for record in annotation_qs:
models.Annotation.objects.create(
record=record,
text=add_annotation_list_form.cleaned_data["text"])
else:
search_form = forms.SearchForm(collection.locations.order_by("pk"),
available_search_fields)
pagination_form = forms.PaginationForm(initial={
'page': '1',
'records_per_page': '15'
})
result_list_form = forms.RecordSearchResultListForm(
collection.locations.order_by("pk"))
# Response formats
if request.POST.get("download_csv"):
keys = display_fields.keys()
f = tempfile.SpooledTemporaryFile()
writer = csv.writer(f, delimiter=",")
writer.writerow(["filename"] + keys)
for record in qs:
writer.writerow([record.filename] +
[getattr(record, key, "") for key in keys])
size = f.tell()
f.seek(0)
response = StreamingHttpResponse(f, content_type="text/csv")
response["Content-Length"] = str(size)
response["Content-Disposition"] = (
'inline; filename="search-%s.csv"' %
(now().replace(microsecond=0, tzinfo=None).isoformat("T")))
return response
elif request.POST.get("download_filenames"):
f = tempfile.SpooledTemporaryFile()
for record in qs:
f.write(record.filename)
f.write("\n")
size = f.tell()
f.seek(0)
response = StreamingHttpResponse(f, content_type="text/plain")
response["Content-Length"] = str(size)
response["Content-Disposition"] = (
'inline; filename="search-%s.txt"' %
(now().replace(microsecond=0, tzinfo=None).isoformat("T")))
return response
else:
# overwrite on purpose
add_annotation_list_form = forms.AddAnnotationListForm()
return render(
request, "inventory/collection/result_list.html", {
"collections": models.Collection.objects.all(),
"search_form": search_form,
"pagination_form": pagination_form,
"result_list_form": result_list_form,
"add_annotation_list_form": add_annotation_list_form,
"collection": collection,
"location": location,
"results": results,
"result_list": result_list,
"result_list_location": result_list_location,
"metadata_fields": display_fields
})
def restart(self, job_id):
job = models.Job.objects.get(id=job_id)
kwargs = job.get_arguments()
with monitor(job.task, **kwargs):
return self._tasks[job.task](**kwargs)
def run(self, task, **kwargs):
name = task if isinstance(task, basestring) else task.task
with monitor(task, **kwargs):
return self._tasks[name](**kwargs)
def alignment_view(request, mission, file_type):
""" Django view function to perform the alignment check.
"""
collection = models.Collection.objects.get(
mission=mission, file_type=file_type
)
config = collection.configuration
records = None
locations = None
locations_with_no_checksum = None
frmt = "html"
if request.method == "POST":
form = forms.AlignmentForm(
collection.locations.order_by("pk"),
config.available_alignment_fields or [],
request.POST
)
pagination_form = forms.PaginationForm(request.POST)
if form.is_valid() and pagination_form.is_valid():
frmt = form.cleaned_data.pop("format")
observer = monitor(
"alignment_check", mission=mission, file_type=file_type,
**form.cleaned_data
)
with observer:
locations, qs = queries.alignment(collection, form.cleaned_data)
page = pagination_form.cleaned_data.pop("page")
per_page = pagination_form.cleaned_data.pop("records_per_page")
if frmt == "html":
records = Paginator(qs, per_page).page(page)
else:
records = qs
locations_with_no_checksum = [
location for location in locations or []
if collection.get_metadata_field_mapping(location).get("checksum") is None
]
else:
form = forms.AlignmentForm(
collection.locations.order_by("pk"),
config.available_alignment_fields or []
)
pagination_form = forms.PaginationForm(
initial={'page': '1', 'records_per_page': '15'}
)
if frmt == "html":
return render(
request, "inventory/collection/alignment.html", {
"collections": models.Collection.objects.all(),
"collection": collection, "alignment_form": form,
"pagination_form": pagination_form, "records": records,
"locations": locations,
"locations_with_no_checksum": locations_with_no_checksum
}
)
elif frmt in ("csv", "tsv"):
f = tempfile.SpooledTemporaryFile()
writer = csv.writer(f, delimiter="," if frmt == "csv" else "\t")
header = ["filename"]
for location in locations:
header.extend([
"checksum %s" % location.url,
"prevalence %s" % location.url
])
header.append("annotations")
writer.writerow(header)
for row in qs:
writer.writerow(
[row["filename"]] +
[item for incidence in row["incidences"] for item in incidence] +
list(row["annotations"])
)
size = f.tell()
f.seek(0)
response = StreamingHttpResponse(f, content_type="text/csv")
response["Content-Length"] = str(size)
response["Content-Disposition"] = (
'inline; filename="alignment-%s.csv"' % (
now().replace(microsecond=0, tzinfo=None).isoformat("T")
)
)
return response
def result_list_view(request, mission, file_type):
"""
"""
collection = models.Collection.objects.get(
mission=mission, file_type=file_type
)
location = None
available_search_fields = get_available_search_fields(collection)
config = collection.configuration
all_choices = dict((("checksum", "Checksum"),) + models.SEARCH_FIELD_CHOICES)
display_fields = SortedDict(
(field_id, all_choices[field_id])
for field_id in config.available_result_list_fields or []
)
results = None
result_list = None
result_list_location = None
if request.method == "POST":
search_form = search_form = forms.SearchForm(
collection.locations.order_by("pk"), available_search_fields,
request.POST
)
pagination_form = forms.PaginationForm(request.POST)
result_list_form = forms.RecordSearchResultListForm(
collection.locations.order_by("pk"), request.POST
)
forms_valid = (
search_form.is_valid() and pagination_form.is_valid() and
result_list_form.is_valid()
)
if forms_valid:
search_data = search_form.cleaned_data
result_list_location_id = result_list_form.cleaned_data[
"result_list_location"
]
# TODO: only scene centre??
footprint_or_scene_centre = search_data.pop(
"area_footprint_or_scene_centre", "footprint"
)
location = collection.locations.get(id=result_list_location_id)
observer = monitor(
"search_results", **search_data
)
with observer:
qs = queries.search(
collection, search_data, location.records.all(),
footprint_or_scene_centre == "footprint"
)
sort = result_list_form.cleaned_data.pop("sort", None)
if sort:
qs = qs.order_by(sort)
page = pagination_form.cleaned_data.pop("page")
per_page = pagination_form.cleaned_data.pop(
"records_per_page"
)
result_list = Paginator(qs, per_page).page(page)
result_list_location = location
# see if we need to create annotations
add_annotation_list_form = forms.AddAnnotationListForm(request.POST)
if add_annotation_list_form.is_valid() and \
add_annotation_list_form.cleaned_data["text"] and \
add_annotation_list_form.cleaned_data["do_add_annotation"]:
ids = map(int, request.POST.getlist(
"add_record_annotation", []
))
if add_annotation_list_form.cleaned_data[
"add_annotations_to_all"]:
annotation_qs = qs
else:
annotation_qs = location.records.filter(pk__in=ids)
for record in annotation_qs:
models.Annotation.objects.create(
record=record,
text=add_annotation_list_form.cleaned_data["text"]
)
else:
search_form = forms.SearchForm(
collection.locations.order_by("pk"), available_search_fields
)
pagination_form = forms.PaginationForm(
initial={'page': '1', 'records_per_page': '15'}
)
result_list_form = forms.RecordSearchResultListForm(
collection.locations.order_by("pk")
)
# Response formats
if request.POST.get("download_csv"):
keys = display_fields.keys()
f = tempfile.SpooledTemporaryFile()
writer = csv.writer(f, delimiter=",")
writer.writerow(
["filename"] + keys
)
for record in qs:
writer.writerow([
record.filename
] + [
getattr(record, key, "")
for key in keys
])
size = f.tell()
f.seek(0)
response = StreamingHttpResponse(f, content_type="text/csv")
response["Content-Length"] = str(size)
response["Content-Disposition"] = (
'inline; filename="search-%s.csv"' % (
now().replace(microsecond=0, tzinfo=None).isoformat("T")
)
)
return response
elif request.POST.get("download_filenames"):
f = tempfile.SpooledTemporaryFile()
for record in qs:
f.write(record.filename)
f.write("\n")
size = f.tell()
f.seek(0)
response = StreamingHttpResponse(f, content_type="text/plain")
response["Content-Length"] = str(size)
response["Content-Disposition"] = (
'inline; filename="search-%s.txt"' % (
now().replace(microsecond=0, tzinfo=None).isoformat("T")
)
)
return response
else:
# overwrite on purpose
add_annotation_list_form = forms.AddAnnotationListForm()
return render(
request, "inventory/collection/result_list.html", {
"collections": models.Collection.objects.all(),
"search_form": search_form,
"pagination_form": pagination_form,
"result_list_form": result_list_form,
"add_annotation_list_form": add_annotation_list_form,
"collection": collection,
"location": location,
"results": results,
"result_list": result_list,
"result_list_location": result_list_location,
"metadata_fields": display_fields
}
)
def search_view(request, mission, file_type):
""" Django view function to perform the collection search.
"""
collection = models.Collection.objects.get(
mission=mission, file_type=file_type
)
available_search_fields = get_available_search_fields(collection)
results = None
if request.method == "POST":
search_form = forms.SearchForm(
collection.locations.order_by("pk"),
available_search_fields, request.POST
)
pagination_form = forms.PaginationForm(request.POST)
result_list_form = forms.RecordSearchResultListForm(
collection.locations.order_by("pk"), request.POST
)
forms_valid = (
search_form.is_valid() and pagination_form.is_valid() and
result_list_form.is_valid()
)
if forms_valid:
search_data = search_form.cleaned_data
location_ids = search_data.pop("locations", ())
# TODO: only scene centre??
footprint_or_scene_centre = search_data.pop(
"area_footprint_or_scene_centre", "footprint"
)
locations = collection.locations.order_by("pk")
if location_ids:
locations = locations.filter(id__in=location_ids)
observer = monitor(
"search_overview", **search_data
)
with observer:
results = []
for location in locations:
qs = queries.search(
collection, search_data, location.records.all(),
footprint_or_scene_centre == "footprint"
)
values = qs.aggregate(
volume=Sum("filesize"), count=Count("filename")
)
results.append((location, values))
else:
search_form = forms.SearchForm(
collection.locations.order_by("pk"), available_search_fields
)
pagination_form = forms.PaginationForm(
initial={'page': '1', 'records_per_page': '15'}
)
result_list_form = forms.RecordSearchResultListForm(
collection.locations.order_by("pk")
)
return render(
request, "inventory/collection/search.html", {
"collections": models.Collection.objects.all(),
"search_form": search_form,
"pagination_form": pagination_form,
"result_list_form": result_list_form,
"collection": collection, "results": results
}
)