def search_view(request, mission, file_type): """ Django view function to perform the collection search. """ collection = models.Collection.objects.get(mission=mission, file_type=file_type) available_search_fields = get_available_search_fields(collection) results = None if request.method == "POST": search_form = forms.SearchForm(collection.locations.order_by("pk"), available_search_fields, request.POST) pagination_form = forms.PaginationForm(request.POST) result_list_form = forms.RecordSearchResultListForm( collection.locations.order_by("pk"), request.POST) forms_valid = (search_form.is_valid() and pagination_form.is_valid() and result_list_form.is_valid()) if forms_valid: search_data = search_form.cleaned_data location_ids = search_data.pop("locations", ()) # TODO: only scene centre?? footprint_or_scene_centre = search_data.pop( "area_footprint_or_scene_centre", "footprint") locations = collection.locations.order_by("pk") if location_ids: locations = locations.filter(id__in=location_ids) observer = monitor("search_overview", **search_data) with observer: results = [] for location in locations: qs = queries.search( collection, search_data, location.records.all(), footprint_or_scene_centre == "footprint") values = qs.aggregate(volume=Sum("filesize"), count=Count("filename")) results.append((location, values)) else: search_form = forms.SearchForm(collection.locations.order_by("pk"), available_search_fields) pagination_form = forms.PaginationForm(initial={ 'page': '1', 'records_per_page': '15' }) result_list_form = forms.RecordSearchResultListForm( collection.locations.order_by("pk")) return render( request, "inventory/collection/search.html", { "collections": models.Collection.objects.all(), "search_form": search_form, "pagination_form": pagination_form, "result_list_form": result_list_form, "collection": collection, "results": results })
def restart(self, job_id): job = models.Job.objects.get(id=job_id) kwargs = job.get_arguments() with monitor(job.task, **kwargs): return self._tasks[job.task](**kwargs)
def run(self, task, **kwargs): name = task if isinstance(task, basestring) else task.task with monitor(task, **kwargs): return self._tasks[name](**kwargs)
def alignment_view(request, mission, file_type): """ Django view function to perform the alignment check. """ collection = models.Collection.objects.get(mission=mission, file_type=file_type) config = collection.configuration records = None locations = None locations_with_no_checksum = None frmt = "html" if request.method == "POST": form = forms.AlignmentForm(collection.locations.order_by("pk"), config.available_alignment_fields or [], request.POST) pagination_form = forms.PaginationForm(request.POST) if form.is_valid() and pagination_form.is_valid(): frmt = form.cleaned_data.pop("format") observer = monitor("alignment_check", mission=mission, file_type=file_type, **form.cleaned_data) with observer: locations, qs = queries.alignment(collection, form.cleaned_data) page = pagination_form.cleaned_data.pop("page") per_page = pagination_form.cleaned_data.pop("records_per_page") if frmt == "html": records = Paginator(qs, per_page).page(page) else: records = qs locations_with_no_checksum = [ location for location in locations or [] if collection.get_metadata_field_mapping(location).get( "checksum") is None ] else: form = forms.AlignmentForm(collection.locations.order_by("pk"), config.available_alignment_fields or []) pagination_form = forms.PaginationForm(initial={ 'page': '1', 'records_per_page': '15' }) if frmt == "html": return render( request, "inventory/collection/alignment.html", { "collections": models.Collection.objects.all(), "collection": collection, "alignment_form": form, "pagination_form": pagination_form, "records": records, "locations": locations, "locations_with_no_checksum": locations_with_no_checksum }) elif frmt in ("csv", "tsv"): f = tempfile.SpooledTemporaryFile() writer = csv.writer(f, delimiter="," if frmt == "csv" else "\t") header = ["filename"] for location in locations: header.extend( ["checksum %s" % location.url, "prevalence %s" % location.url]) header.append("annotations") writer.writerow(header) for row in qs: writer.writerow([row["filename"]] + [ item for incidence in row["incidences"] for item in incidence ] + list(row["annotations"])) size = f.tell() f.seek(0) response = StreamingHttpResponse(f, content_type="text/csv") response["Content-Length"] = str(size) response["Content-Disposition"] = ( 'inline; filename="alignment-%s.csv"' % (now().replace(microsecond=0, tzinfo=None).isoformat("T"))) return response
def result_list_view(request, mission, file_type): """ """ collection = models.Collection.objects.get(mission=mission, file_type=file_type) location = None available_search_fields = get_available_search_fields(collection) config = collection.configuration all_choices = dict((("checksum", "Checksum"), ) + models.SEARCH_FIELD_CHOICES) display_fields = SortedDict( (field_id, all_choices[field_id]) for field_id in config.available_result_list_fields or []) results = None result_list = None result_list_location = None if request.method == "POST": search_form = search_form = forms.SearchForm( collection.locations.order_by("pk"), available_search_fields, request.POST) pagination_form = forms.PaginationForm(request.POST) result_list_form = forms.RecordSearchResultListForm( collection.locations.order_by("pk"), request.POST) forms_valid = (search_form.is_valid() and pagination_form.is_valid() and result_list_form.is_valid()) if forms_valid: search_data = search_form.cleaned_data result_list_location_id = result_list_form.cleaned_data[ "result_list_location"] # TODO: only scene centre?? footprint_or_scene_centre = search_data.pop( "area_footprint_or_scene_centre", "footprint") location = collection.locations.get(id=result_list_location_id) observer = monitor("search_results", **search_data) with observer: qs = queries.search(collection, search_data, location.records.all(), footprint_or_scene_centre == "footprint") sort = result_list_form.cleaned_data.pop("sort", None) if sort: qs = qs.order_by(sort) page = pagination_form.cleaned_data.pop("page") per_page = pagination_form.cleaned_data.pop("records_per_page") result_list = Paginator(qs, per_page).page(page) result_list_location = location # see if we need to create annotations add_annotation_list_form = forms.AddAnnotationListForm( request.POST) if add_annotation_list_form.is_valid() and \ add_annotation_list_form.cleaned_data["text"] and \ add_annotation_list_form.cleaned_data["do_add_annotation"]: ids = map(int, request.POST.getlist("add_record_annotation", [])) if add_annotation_list_form.cleaned_data[ "add_annotations_to_all"]: annotation_qs = qs else: annotation_qs = location.records.filter(pk__in=ids) for record in annotation_qs: models.Annotation.objects.create( record=record, text=add_annotation_list_form.cleaned_data["text"]) else: search_form = forms.SearchForm(collection.locations.order_by("pk"), available_search_fields) pagination_form = forms.PaginationForm(initial={ 'page': '1', 'records_per_page': '15' }) result_list_form = forms.RecordSearchResultListForm( collection.locations.order_by("pk")) # Response formats if request.POST.get("download_csv"): keys = display_fields.keys() f = tempfile.SpooledTemporaryFile() writer = csv.writer(f, delimiter=",") writer.writerow(["filename"] + keys) for record in qs: writer.writerow([record.filename] + [getattr(record, key, "") for key in keys]) size = f.tell() f.seek(0) response = StreamingHttpResponse(f, content_type="text/csv") response["Content-Length"] = str(size) response["Content-Disposition"] = ( 'inline; filename="search-%s.csv"' % (now().replace(microsecond=0, tzinfo=None).isoformat("T"))) return response elif request.POST.get("download_filenames"): f = tempfile.SpooledTemporaryFile() for record in qs: f.write(record.filename) f.write("\n") size = f.tell() f.seek(0) response = StreamingHttpResponse(f, content_type="text/plain") response["Content-Length"] = str(size) response["Content-Disposition"] = ( 'inline; filename="search-%s.txt"' % (now().replace(microsecond=0, tzinfo=None).isoformat("T"))) return response else: # overwrite on purpose add_annotation_list_form = forms.AddAnnotationListForm() return render( request, "inventory/collection/result_list.html", { "collections": models.Collection.objects.all(), "search_form": search_form, "pagination_form": pagination_form, "result_list_form": result_list_form, "add_annotation_list_form": add_annotation_list_form, "collection": collection, "location": location, "results": results, "result_list": result_list, "result_list_location": result_list_location, "metadata_fields": display_fields })
def alignment_view(request, mission, file_type): """ Django view function to perform the alignment check. """ collection = models.Collection.objects.get( mission=mission, file_type=file_type ) config = collection.configuration records = None locations = None locations_with_no_checksum = None frmt = "html" if request.method == "POST": form = forms.AlignmentForm( collection.locations.order_by("pk"), config.available_alignment_fields or [], request.POST ) pagination_form = forms.PaginationForm(request.POST) if form.is_valid() and pagination_form.is_valid(): frmt = form.cleaned_data.pop("format") observer = monitor( "alignment_check", mission=mission, file_type=file_type, **form.cleaned_data ) with observer: locations, qs = queries.alignment(collection, form.cleaned_data) page = pagination_form.cleaned_data.pop("page") per_page = pagination_form.cleaned_data.pop("records_per_page") if frmt == "html": records = Paginator(qs, per_page).page(page) else: records = qs locations_with_no_checksum = [ location for location in locations or [] if collection.get_metadata_field_mapping(location).get("checksum") is None ] else: form = forms.AlignmentForm( collection.locations.order_by("pk"), config.available_alignment_fields or [] ) pagination_form = forms.PaginationForm( initial={'page': '1', 'records_per_page': '15'} ) if frmt == "html": return render( request, "inventory/collection/alignment.html", { "collections": models.Collection.objects.all(), "collection": collection, "alignment_form": form, "pagination_form": pagination_form, "records": records, "locations": locations, "locations_with_no_checksum": locations_with_no_checksum } ) elif frmt in ("csv", "tsv"): f = tempfile.SpooledTemporaryFile() writer = csv.writer(f, delimiter="," if frmt == "csv" else "\t") header = ["filename"] for location in locations: header.extend([ "checksum %s" % location.url, "prevalence %s" % location.url ]) header.append("annotations") writer.writerow(header) for row in qs: writer.writerow( [row["filename"]] + [item for incidence in row["incidences"] for item in incidence] + list(row["annotations"]) ) size = f.tell() f.seek(0) response = StreamingHttpResponse(f, content_type="text/csv") response["Content-Length"] = str(size) response["Content-Disposition"] = ( 'inline; filename="alignment-%s.csv"' % ( now().replace(microsecond=0, tzinfo=None).isoformat("T") ) ) return response
def result_list_view(request, mission, file_type): """ """ collection = models.Collection.objects.get( mission=mission, file_type=file_type ) location = None available_search_fields = get_available_search_fields(collection) config = collection.configuration all_choices = dict((("checksum", "Checksum"),) + models.SEARCH_FIELD_CHOICES) display_fields = SortedDict( (field_id, all_choices[field_id]) for field_id in config.available_result_list_fields or [] ) results = None result_list = None result_list_location = None if request.method == "POST": search_form = search_form = forms.SearchForm( collection.locations.order_by("pk"), available_search_fields, request.POST ) pagination_form = forms.PaginationForm(request.POST) result_list_form = forms.RecordSearchResultListForm( collection.locations.order_by("pk"), request.POST ) forms_valid = ( search_form.is_valid() and pagination_form.is_valid() and result_list_form.is_valid() ) if forms_valid: search_data = search_form.cleaned_data result_list_location_id = result_list_form.cleaned_data[ "result_list_location" ] # TODO: only scene centre?? footprint_or_scene_centre = search_data.pop( "area_footprint_or_scene_centre", "footprint" ) location = collection.locations.get(id=result_list_location_id) observer = monitor( "search_results", **search_data ) with observer: qs = queries.search( collection, search_data, location.records.all(), footprint_or_scene_centre == "footprint" ) sort = result_list_form.cleaned_data.pop("sort", None) if sort: qs = qs.order_by(sort) page = pagination_form.cleaned_data.pop("page") per_page = pagination_form.cleaned_data.pop( "records_per_page" ) result_list = Paginator(qs, per_page).page(page) result_list_location = location # see if we need to create annotations add_annotation_list_form = forms.AddAnnotationListForm(request.POST) if add_annotation_list_form.is_valid() and \ add_annotation_list_form.cleaned_data["text"] and \ add_annotation_list_form.cleaned_data["do_add_annotation"]: ids = map(int, request.POST.getlist( "add_record_annotation", [] )) if add_annotation_list_form.cleaned_data[ "add_annotations_to_all"]: annotation_qs = qs else: annotation_qs = location.records.filter(pk__in=ids) for record in annotation_qs: models.Annotation.objects.create( record=record, text=add_annotation_list_form.cleaned_data["text"] ) else: search_form = forms.SearchForm( collection.locations.order_by("pk"), available_search_fields ) pagination_form = forms.PaginationForm( initial={'page': '1', 'records_per_page': '15'} ) result_list_form = forms.RecordSearchResultListForm( collection.locations.order_by("pk") ) # Response formats if request.POST.get("download_csv"): keys = display_fields.keys() f = tempfile.SpooledTemporaryFile() writer = csv.writer(f, delimiter=",") writer.writerow( ["filename"] + keys ) for record in qs: writer.writerow([ record.filename ] + [ getattr(record, key, "") for key in keys ]) size = f.tell() f.seek(0) response = StreamingHttpResponse(f, content_type="text/csv") response["Content-Length"] = str(size) response["Content-Disposition"] = ( 'inline; filename="search-%s.csv"' % ( now().replace(microsecond=0, tzinfo=None).isoformat("T") ) ) return response elif request.POST.get("download_filenames"): f = tempfile.SpooledTemporaryFile() for record in qs: f.write(record.filename) f.write("\n") size = f.tell() f.seek(0) response = StreamingHttpResponse(f, content_type="text/plain") response["Content-Length"] = str(size) response["Content-Disposition"] = ( 'inline; filename="search-%s.txt"' % ( now().replace(microsecond=0, tzinfo=None).isoformat("T") ) ) return response else: # overwrite on purpose add_annotation_list_form = forms.AddAnnotationListForm() return render( request, "inventory/collection/result_list.html", { "collections": models.Collection.objects.all(), "search_form": search_form, "pagination_form": pagination_form, "result_list_form": result_list_form, "add_annotation_list_form": add_annotation_list_form, "collection": collection, "location": location, "results": results, "result_list": result_list, "result_list_location": result_list_location, "metadata_fields": display_fields } )
def search_view(request, mission, file_type): """ Django view function to perform the collection search. """ collection = models.Collection.objects.get( mission=mission, file_type=file_type ) available_search_fields = get_available_search_fields(collection) results = None if request.method == "POST": search_form = forms.SearchForm( collection.locations.order_by("pk"), available_search_fields, request.POST ) pagination_form = forms.PaginationForm(request.POST) result_list_form = forms.RecordSearchResultListForm( collection.locations.order_by("pk"), request.POST ) forms_valid = ( search_form.is_valid() and pagination_form.is_valid() and result_list_form.is_valid() ) if forms_valid: search_data = search_form.cleaned_data location_ids = search_data.pop("locations", ()) # TODO: only scene centre?? footprint_or_scene_centre = search_data.pop( "area_footprint_or_scene_centre", "footprint" ) locations = collection.locations.order_by("pk") if location_ids: locations = locations.filter(id__in=location_ids) observer = monitor( "search_overview", **search_data ) with observer: results = [] for location in locations: qs = queries.search( collection, search_data, location.records.all(), footprint_or_scene_centre == "footprint" ) values = qs.aggregate( volume=Sum("filesize"), count=Count("filename") ) results.append((location, values)) else: search_form = forms.SearchForm( collection.locations.order_by("pk"), available_search_fields ) pagination_form = forms.PaginationForm( initial={'page': '1', 'records_per_page': '15'} ) result_list_form = forms.RecordSearchResultListForm( collection.locations.order_by("pk") ) return render( request, "inventory/collection/search.html", { "collections": models.Collection.objects.all(), "search_form": search_form, "pagination_form": pagination_form, "result_list_form": result_list_form, "collection": collection, "results": results } )