def search(request, id=None, name=None, selected=False, json=False): collection = id and get_object_or_404(filter_by_access(request.user, Collection), id=id) or None if request.method == "POST": update_record_selection(request) # redirect to get request with updated parameters q = request.GET.copy() q.update(request.POST) q = clean_record_selection_vars(q) for i, v in q.items(): if i != 'c': q[i] = v # replace multiple values with last one except for criteria ('c') q.pop('v.x', None) q.pop('v.y', None) q.pop('x', None) q.pop('y', None) return HttpResponseRedirect(request.path + '?' + q.urlencode()) # get parameters relevant for search criteria = request.GET.getlist('c') remove = request.GET.get('rem', None) if remove and remove in criteria: criteria.remove(remove) keywords = request.GET.get('kw', '') # get parameters relevant for view viewmode = request.GET.get('v', 'thumb') if viewmode == 'list': pagesize = max(min(safe_int(request.GET.get('ps', '50'), 50), 100), 5) else: pagesize = max(min(safe_int(request.GET.get('ps', '30'), 30), 50), 5) page = safe_int(request.GET.get('page', '1'), 1) sort = request.GET.get('s', 'title_sort').lower() if not sort.endswith(" asc") and not sort.endswith(" desc"): sort += " asc" orquery = request.GET.get('or', None) user = request.user if request.GET.has_key('action'): page = safe_int(request.GET.get('op', '1'), 1) if selected: selected = request.session.get('selected_records', ()) (hits, records, search_facets, orfacet, query, fields) = run_search(user, collection, criteria, keywords, sort, page, pagesize, orquery, selected, remove, produce_facets=False) if json: return (hits, records, viewmode) if collection: url = reverse('solr-search-collection', kwargs={'id': collection.id, 'name': collection.name}) furl = reverse('solr-search-collection-facets', kwargs={'id': collection.id, 'name': collection.name}) elif hasattr(selected, '__len__'): url = reverse('solr-selected') furl = reverse('solr-selected-facets') else: url = reverse('solr-search') furl = reverse('solr-search-facets') q = request.GET.copy() q = clean_record_selection_vars(q) q.pop('or', None) q.pop('rem', None) q.pop('action', None) q.pop('page', None) q.pop('op', None) q.pop('v.x', None) q.pop('v.y', None) q.pop('x', None) q.pop('y', None) q['s'] = q.get('s', sort) q['v'] = q.get('v', 'thumb') q.setlist('c', criteria) hiddenfields = [('op', page)] #for f in q: # if f != 'kw': # for l in q.getlist(f): # hiddenfields.append((f, l)) qurl = q.urlencode() q.setlist('c', filter(lambda c: c != orquery, criteria)) qurl_orquery = q.urlencode() limit_url = "%s?%s%s" % (url, qurl, qurl and '&' or '') limit_url_orquery = "%s?%s%s" % (url, qurl_orquery, qurl_orquery and '&' or '') facets_url = "%s?%s%s" % (furl, qurl, qurl and '&' or '') form_url = "%s?%s" % (url, q.urlencode()) prev_page_url = None next_page_url = None if page > 1: q['page'] = page - 1 prev_page_url = "%s?%s" % (url, q.urlencode()) if page < (hits - 1) / pagesize + 1: q['page'] = page + 1 next_page_url = "%s?%s" % (url, q.urlencode()) def readable_criteria(c): (f, o) = c.split(':', 1) negated = f.startswith('-') f = f[1 if negated else 0:] if search_facets.has_key(f): return dict(facet=c, term=search_facets[f].display_value(o), label=search_facets[f].label, negated=negated, or_available=not negated and search_facets[f].or_available()) else: return dict(facet=c, term=o, label='Unknown criteria', negated=negated, or_available=False) def reduce_federated_search_query(q, c): (f, o) = c.split(':', 1) if f.startswith('-') or not search_facets.has_key(f): # can't negate in federated search return q v = search_facets[f].federated_search_query(o) return v if not q else '%s %s' % (q, v) mode, ids = get_collection_visibility_preferences(user) hash = calculate_hash(getattr(user, 'id', 0), collection, criteria, keywords, selected, remove, mode, str(ids), ) print hash facets = cache.get('search_facets_html_%s' % hash) sort = sort.startswith('random') and 'random' or sort.split()[0] sort = sort.endswith('_sort') and sort[:-5] or sort federated_search_query = reduce(reduce_federated_search_query, criteria, keywords) federated_search = sidebar_api_raw( request, federated_search_query, cached_only=True) if federated_search_query else None return render_to_response('results.html', {'criteria': map(readable_criteria, criteria), 'query': query, 'keywords': keywords, 'hiddenfields': hiddenfields, 'records': records, 'hits': hits, 'page': page, 'pages': (hits - 1) / pagesize + 1, 'pagesize': pagesize, 'prev_page': prev_page_url, 'next_page': next_page_url, 'reset_url': url, 'form_url': form_url, 'limit_url': limit_url, 'limit_url_orquery': limit_url_orquery, 'facets': facets, 'facets_url': facets_url, 'orfacet': orfacet, 'orquery': orquery, 'sort': sort, 'random': random.random(), 'viewmode': viewmode, 'federated_sources': bool(available_federated_sources()), 'federated_search': federated_search, 'federated_search_query': federated_search_query, 'pagination_helper': [None] * hits, 'has_record_created_criteria': any(f.startswith('created:') for f in criteria), 'has_last_modified_criteria': any(f.startswith('modified:') for f in criteria), }, context_instance=RequestContext(request))
def search(request, id=None, name=None, selected=False, json=False): collection = id and get_object_or_404(filter_by_access(request.user, Collection), id=id) or None update_record_selection(request) # get parameters relevant for search criteria = request.GET.getlist('c') remove = request.GET.get('rem', None) if remove: criteria.remove(remove) keywords = request.GET.get('kw', '') # get parameters relevant for view viewmode = request.GET.get('v', 'thumb') if viewmode == 'list': pagesize = max(min(safe_int(request.GET.get('ps', '50'), 50), 100), 5) else: pagesize = max(min(safe_int(request.GET.get('ps', '30'), 30), 50), 5) page = safe_int(request.GET.get('p', '1'), 1) sort = request.GET.get('s', 'title_sort').lower() if not sort.endswith(" asc") and not sort.endswith(" desc"): sort += " asc" orquery = request.GET.get('or', None) user = request.user if request.GET.has_key('action'): page = safe_int(request.GET.get('op', '1'), 1) if selected: selected = request.session.get('selected_records', ()) (hits, records, search_facets, orfacet, query, fields) = run_search(user, collection, criteria, keywords, sort, page, pagesize, orquery, selected, remove, produce_facets=False) if json: return (hits, records, viewmode) if collection: url = reverse('solr-search-collection', kwargs={'id': collection.id, 'name': collection.name}) furl = reverse('solr-search-collection-facets', kwargs={'id': collection.id, 'name': collection.name}) elif hasattr(selected, '__len__'): url = reverse('solr-selected') furl = reverse('solr-selected-facets') else: url = reverse('solr-search') furl = reverse('solr-search-facets') q = request.GET.copy() q = clean_record_selection_vars(q) q.pop('or', None) q.pop('rem', None) q.pop('action', None) q.pop('p', None) q.pop('op', None) q.pop('v.x', None) q.pop('v.y', None) q['s'] = q.get('s', sort) q['v'] = q.get('v', 'thumb') q.setlist('c', criteria) hiddenfields = [('op', page)] for f in q: if f != 'kw': for l in q.getlist(f): hiddenfields.append((f, l)) qurl = q.urlencode() q.setlist('c', filter(lambda c: c != orquery, criteria)) qurl_orquery = q.urlencode() limit_url = "%s?%s%s" % (url, qurl, qurl and '&' or '') limit_url_orquery = "%s?%s%s" % (url, qurl_orquery, qurl_orquery and '&' or '') facets_url = "%s?%s%s" % (furl, qurl, qurl and '&' or '') prev_page_url = None next_page_url = None if page > 1: q['p'] = page - 1 prev_page_url = "%s?%s" % (url, q.urlencode()) if page < (hits - 1) / pagesize + 1: q['p'] = page + 1 next_page_url = "%s?%s" % (url, q.urlencode()) q.pop('s', None) form_url = "%s?%s" % (url, q.urlencode()) def readable_criteria(c): (f, o) = c.split(':', 1) negated = f.startswith('-') f = f[1 if negated else 0:] if search_facets.has_key(f): return dict(facet=c, term=search_facets[f].display_value(o), label=search_facets[f].label, negated=negated, or_available=not negated and search_facets[f].or_available()) else: return dict(facet=c, term=o, label='Unknown criteria', negated=negated, or_available=False) def federated_search_query(q, c): (f, o) = c.split(':', 1) if f.startswith('-') or not search_facets.has_key(f): # can't negate in federated search return q v = search_facets[f].federated_search_query(o) return v if not q else '%s %s' % (q, v) # sort facets by label facets = sorted(search_facets.values(), key=lambda f: f.label) # clean facet items for f in facets: f.clean_result(hits) # remove facets with only no filter options facets = filter(lambda f: len(f.facets) > 0, facets) sort = sort.startswith('random') and 'random' or sort.split()[0] sort = sort.endswith('_sort') and sort[:-5] or sort return render_to_response('results.html', {'criteria': map(readable_criteria, criteria), 'query': query, 'keywords': keywords, 'hiddenfields': hiddenfields, 'records': records, 'hits': hits, 'page': page, 'pages': (hits - 1) / pagesize + 1, 'prev_page': prev_page_url, 'next_page': next_page_url, 'reset_url': url, 'form_url': form_url, 'limit_url': limit_url, 'limit_url_orquery': limit_url_orquery, 'facets': facets, 'facets_url': facets_url, 'orfacet': orfacet, 'orquery': orquery, 'sort': sort, 'random': random.random(), 'viewmode': viewmode, 'federated_search_query': reduce(federated_search_query, criteria, keywords), }, context_instance=RequestContext(request))
def data_import_file(request, file): available_collections = filter_by_access(request.user, Collection) writable_collection_ids = list(filter_by_access(request.user, Collection, write=True).values_list('id', flat=True)) if not available_collections: raise Http404 available_fieldsets = FieldSet.for_user(request.user) def _get_fields(): return Field.objects.select_related('standard').all().order_by('standard', 'name') def _field_choices(): fsf = list(FieldSetField.objects.select_related('fieldset', 'field').exclude(fieldset__name__startswith='browse-collection-').exclude(fieldset__name='facet-fields').order_by('fieldset__name', 'order', 'field__label')) grouped = {} for f in fsf: grouped.setdefault((f.fieldset.title, f.fieldset.id), []).append(f.field) others = list(Field.objects.exclude(id__in=[f.field.id for f in fsf]).order_by('label').values_list('id', 'label')) choices = [('', '-' * 10)] + [(set[0], [(f.id, f.label) for f in fields]) for set, fields in sorted(grouped.iteritems(), key=lambda s: s[0][0])] if others: choices.append(('Others', others)) return choices #def _field_choices(): # grouped = {} # for f in _get_fields(): # grouped.setdefault(f.standard and f.standard.title or 'Other', []).append(f) # return [('0', '[do not import]'), ('-1', '[new field]')] + \ # [(g, [(f.id, f.label) for f in grouped[g]]) for g in grouped] class ImportOptionsForm(forms.Form): separator = forms.CharField(required=False) collections = forms.MultipleChoiceField(choices=((c.id, '%s%s' % ('*' if c.id in writable_collection_ids else '', c.title)) for c in sorted(available_collections, key=lambda c: c.title)), widget=forms.CheckboxSelectMultiple) fieldset = FieldSetChoiceField(user=request.user, default_label='any') #forms.ChoiceField(choices=[(0, 'any')] + [(f.id, f.title) for f in available_fieldsets], required=False) update = forms.BooleanField(label='Update existing records', initial=True, required=False) add = forms.BooleanField(label='Add new records', initial=True, required=False) test = forms.BooleanField(label='Test import only', initial=False, required=False) personal = forms.BooleanField(label='Personal records', initial=False, required=False) def clean(self): cleaned_data = self.cleaned_data if any(self.errors): return cleaned_data personal = cleaned_data['personal'] if not personal: for c in map(int, cleaned_data['collections']): if not c in writable_collection_ids: self._errors['collections'] = ErrorList(["Can only add personal records to selected collections"]) del cleaned_data['collections'] return cleaned_data return cleaned_data class MappingForm(forms.Form): fieldname = forms.CharField(widget=DisplayOnlyTextWidget, required=False) mapping = forms.ChoiceField(choices=_field_choices(), required=False) separate = forms.BooleanField(required=False) label = forms.CharField(required=False) hidden = forms.BooleanField(required=False) class BaseMappingFormSet(forms.formsets.BaseFormSet): def clean(self): if any(self.errors): return _dcidentifier = Field.objects.get(name='identifier', standard__prefix='dc') _identifier_ids = list(_dcidentifier.get_equivalent_fields().values_list('id', flat=True)) + [_dcidentifier.id] for i in range(self.total_form_count()): if self.forms[i].cleaned_data['mapping'] and \ (int(self.forms[i].cleaned_data['mapping']) in _identifier_ids): return raise forms.ValidationError, "At least one field must be mapped to an identifier field." MappingFormSet = formset_factory(MappingForm, extra=0, formset=BaseMappingFormSet, can_order=True) def analyze(collections=None, separator=None, separate_fields=None, fieldset=None): try: with open(os.path.join(_get_scratch_dir(), _get_filename(request, file)), 'rU') as csvfile: imp = SpreadsheetImport(csvfile, collections, separator=separator, separate_fields=separate_fields, preferred_fieldset=fieldset) return imp, imp.analyze() except IOError: raise Http404() if request.method == 'POST': form = ImportOptionsForm(request.POST) mapping_formset = MappingFormSet(request.POST, prefix='m') if form.is_valid() and mapping_formset.is_valid(): imp, preview_rows = analyze(available_collections.filter(id__in=form.cleaned_data['collections']), form.cleaned_data['separator'], dict((f.cleaned_data['fieldname'], f.cleaned_data['separate']) for f in mapping_formset.forms), available_fieldsets.get(id=form.cleaned_data['fieldset']) if int(form.cleaned_data.get('fieldset') or 0) else None) store_settings(request.user, 'data_import_file_%s' % imp.field_hash, simplejson.dumps(dict(options=form.cleaned_data, mapping=mapping_formset.cleaned_data))) if request.POST.get('import_button'): j = JobInfo.objects.create(owner=request.user, func='csvimport', arg=simplejson.dumps(dict( file=_get_filename(request, file), separator=form.cleaned_data['separator'], collections=map(int, form.cleaned_data['collections']), update=form.cleaned_data['update'], add=form.cleaned_data['add'], test=form.cleaned_data['test'], personal=form.cleaned_data['personal'], fieldset=form.cleaned_data['fieldset'], mapping=dict((f.cleaned_data['fieldname'], int(f.cleaned_data['mapping'])) for f in mapping_formset.forms if f.cleaned_data['mapping']), separate_fields=dict((f.cleaned_data['fieldname'], f.cleaned_data['separate']) for f in mapping_formset.forms), labels=dict((f.cleaned_data['fieldname'], f.cleaned_data['label']) for f in mapping_formset.forms), order=dict((f.cleaned_data['fieldname'], safe_int(f.cleaned_data['ORDER'], 0)) for f in mapping_formset.forms), hidden=dict((f.cleaned_data['fieldname'], f.cleaned_data['hidden']) for f in mapping_formset.forms), ) )) j.run() request.user.message_set.create(message='Import job has been submitted.') return HttpResponseRedirect("%s?highlight=%s" % (reverse('workers-jobs'), j.id)) else: imp, preview_rows = analyze() else: imp, preview_rows = analyze() # try to load previously stored settings key = 'data_import_file_%s' % imp.field_hash values = load_settings(request.user, key) if values.has_key(key): value = simplejson.loads(values[key][0]) mapping = value['mapping'] options = value['options'] else: mapping = [dict(fieldname=f, mapping=v.id if v else 0, separate=True) for f, v in imp.mapping.iteritems()] options = None mapping = sorted(mapping, key=lambda m: m['fieldname']) mapping_formset = MappingFormSet(initial=mapping, prefix='m') form = ImportOptionsForm(initial=options) return render_to_response('data_import_file.html', {'form': form, 'preview_rows': preview_rows, 'mapping_formset': mapping_formset, 'writable_collections': bool(writable_collection_ids), }, context_instance=RequestContext(request))
def search(request, id=None, name=None, selected=False, json=False): collection = get_object_or_404(filter_by_access(request.user, Collection), id=id) if id else None if request.method == "POST": update_record_selection(request) # redirect to get request with updated parameters q = request.GET.copy() q.update(request.POST) q = clean_record_selection_vars(q) for i, v in q.items(): if i != "c": # replace multiple values with last one # except for criteria ('c') q[i] = v q.pop("v.x", None) q.pop("v.y", None) q.pop("x", None) q.pop("y", None) return HttpResponseRedirect(request.path + "?" + q.urlencode()) # get parameters relevant for search criteria = request.GET.getlist("c") remove = request.GET.get("rem", None) if remove and remove in criteria: criteria.remove(remove) keywords = request.GET.get("kw", "") # get parameters relevant for view viewmode = request.GET.get("v", "thumb") if viewmode == "list": pagesize = max(min(safe_int(request.GET.get("ps", "50"), 50), 100), 5) else: pagesize = max(min(safe_int(request.GET.get("ps", "30"), 30), 50), 5) page = safe_int(request.GET.get("page", "1"), 1) sort = request.GET.get("s", "title_sort").lower() if not sort.endswith(" asc") and not sort.endswith(" desc"): sort += " asc" orquery = request.GET.get("or", None) user = request.user if "action" in request.GET: page = safe_int(request.GET.get("op", "1"), 1) if selected: selected = request.session.get("selected_records", ()) hits, records, search_facets, orfacet, query, fields = run_search( user, collection, criteria, keywords, sort, page, pagesize, orquery, selected, remove, produce_facets=False ) if json: return (hits, records, viewmode) if collection: url = reverse("solr-search-collection", kwargs={"id": collection.id, "name": collection.name}) furl = reverse("solr-search-collection-facets", kwargs={"id": collection.id, "name": collection.name}) elif hasattr(selected, "__len__"): url = reverse("solr-selected") furl = reverse("solr-selected-facets") else: url = reverse("solr-search") furl = reverse("solr-search-facets") q = request.GET.copy() q = clean_record_selection_vars(q) q.pop("or", None) q.pop("rem", None) q.pop("action", None) q.pop("page", None) q.pop("op", None) q.pop("v.x", None) q.pop("v.y", None) q.pop("x", None) q.pop("y", None) q["s"] = q.get("s", sort) q["v"] = q.get("v", "thumb") q.setlist("c", criteria) hiddenfields = [("op", page)] qurl = q.urlencode() q.setlist("c", filter(lambda c: c != orquery, criteria)) qurl_orquery = q.urlencode() limit_url = "%s?%s%s" % (url, qurl, qurl and "&" or "") limit_url_orquery = "%s?%s%s" % (url, qurl_orquery, qurl_orquery and "&" or "") facets_url = "%s?%s%s" % (furl, qurl, qurl and "&" or "") form_url = "%s?%s" % (url, q.urlencode()) prev_page_url = None next_page_url = None if page > 1: q["page"] = page - 1 prev_page_url = "%s?%s" % (url, q.urlencode()) if page < (hits - 1) / pagesize + 1: q["page"] = page + 1 next_page_url = "%s?%s" % (url, q.urlencode()) def readable_criteria(c): (f, o) = c.split(":", 1) negated = f.startswith("-") f = f[1 if negated else 0 :] if f in search_facets: return dict( facet=c, term=search_facets[f].display_value(o), label=search_facets[f].label, negated=negated, or_available=not negated and search_facets[f].or_available(), ) else: return dict(facet=c, term=o, label="Unknown criteria", negated=negated, or_available=False) def reduce_federated_search_query(q, c): (f, o) = c.split(":", 1) if f.startswith("-") or f not in search_facets: # can't negate in federated search return q v = search_facets[f].federated_search_query(o) return v if not q else "%s %s" % (q, v) mode, ids = get_collection_visibility_preferences(user) hash = calculate_hash(getattr(user, "id", 0), collection, criteria, keywords, selected, remove, mode, str(ids)) facets = cache.get("search_facets_html_%s" % hash) sort = sort.startswith("random") and "random" or sort.split()[0] sort = sort.endswith("_sort") and sort[:-5] or sort federated_search_query = reduce(reduce_federated_search_query, criteria, keywords) federated_search = ( sidebar_api_raw(request, federated_search_query, cached_only=True) if federated_search_query else None ) return render_to_response( "results.html", { "criteria": map(readable_criteria, criteria), "query": query, "keywords": keywords, "hiddenfields": hiddenfields, "records": records, "hits": hits, "page": page, "pages": (hits - 1) / pagesize + 1, "pagesize": pagesize, "prev_page": prev_page_url, "next_page": next_page_url, "reset_url": url, "form_url": form_url, "limit_url": limit_url, "limit_url_orquery": limit_url_orquery, "facets": facets, "facets_url": facets_url, "orfacet": orfacet, "orquery": orquery, "sort": sort, "random": random.random(), "viewmode": viewmode, "federated_sources": bool(available_federated_sources(request.user)), "federated_search": federated_search, "federated_search_query": federated_search_query, "pagination_helper": [None] * hits, "has_record_created_criteria": any(f.startswith("created:") for f in criteria), "has_last_modified_criteria": any(f.startswith("modified:") for f in criteria), }, context_instance=RequestContext(request), )
def search(request, id=None, name=None, selected=False, json=False): collection = get_object_or_404( filter_by_access(request.user, Collection), id=id) if id else None if request.method == "POST": update_record_selection(request) # redirect to get request with updated parameters q = request.GET.copy() q.update(request.POST) q = clean_record_selection_vars(q) for i, v in q.items(): if i != 'c': # replace multiple values with last one # except for criteria ('c') q[i] = v q.pop('v.x', None) q.pop('v.y', None) q.pop('x', None) q.pop('y', None) return HttpResponseRedirect(request.path + '?' + q.urlencode()) # get parameters relevant for search criteria = request.GET.getlist('c') remove = request.GET.get('rem', None) if remove and remove in criteria: criteria.remove(remove) keywords = request.GET.get('kw', '') # get parameters relevant for view viewmode = request.GET.get('v', 'thumb') if viewmode == 'list': pagesize = max(min(safe_int(request.GET.get('ps', '50'), 50), 100), 5) else: pagesize = max(min(safe_int(request.GET.get('ps', '30'), 30), 50), 5) page = safe_int(request.GET.get('page', '1'), 1) sort = request.GET.get('s', 'title_sort').lower() if not sort.endswith(" asc") and not sort.endswith(" desc"): sort += " asc" orquery = request.GET.get('or', None) user = request.user if 'action' in request.GET: page = safe_int(request.GET.get('op', '1'), 1) if selected: selected = request.session.get('selected_records', ()) hits, records, search_facets, orfacet, query, fields = run_search( user, collection, criteria, keywords, sort, page, pagesize, orquery, selected, remove, produce_facets=False) if json: return (hits, records, viewmode) if collection: url = reverse('solr-search-collection', kwargs={'id': collection.id, 'name': collection.name}) furl = reverse('solr-search-collection-facets', kwargs={'id': collection.id, 'name': collection.name}) elif hasattr(selected, '__len__'): url = reverse('solr-selected') furl = reverse('solr-selected-facets') else: url = reverse('solr-search') furl = reverse('solr-search-facets') q = request.GET.copy() q = clean_record_selection_vars(q) q.pop('or', None) q.pop('rem', None) q.pop('action', None) q.pop('page', None) q.pop('op', None) q.pop('v.x', None) q.pop('v.y', None) q.pop('x', None) q.pop('y', None) q['s'] = q.get('s', sort) q['v'] = q.get('v', 'thumb') q.setlist('c', criteria) hiddenfields = [('op', page)] qurl = q.urlencode() q.setlist('c', filter(lambda c: c != orquery, criteria)) qurl_orquery = q.urlencode() limit_url = "%s?%s%s" % (url, qurl, qurl and '&' or '') limit_url_orquery = "%s?%s%s" % ( url, qurl_orquery, qurl_orquery and '&' or '') facets_url = "%s?%s%s" % (furl, qurl, qurl and '&' or '') form_url = "%s?%s" % (url, q.urlencode()) prev_page_url = None next_page_url = None if page > 1: q['page'] = page - 1 prev_page_url = "%s?%s" % (url, q.urlencode()) if page < (hits - 1) / pagesize + 1: q['page'] = page + 1 next_page_url = "%s?%s" % (url, q.urlencode()) def readable_criteria(c): (f, o) = c.split(':', 1) negated = f.startswith('-') f = f[1 if negated else 0:] if f in search_facets: return dict( facet=c, term=search_facets[f].display_value(o), label=search_facets[f].label, negated=negated, or_available=not negated and search_facets[f].or_available(), ) else: return dict( facet=c, term=o, label='Unknown criteria', negated=negated, or_available=False, ) def reduce_federated_search_query(q, c): (f, o) = c.split(':', 1) if f.startswith('-') or f not in search_facets: # can't negate in federated search return q v = search_facets[f].federated_search_query(o) return v if not q else '%s %s' % (q, v) mode, ids = get_collection_visibility_preferences(user) hash = calculate_hash(getattr(user, 'id', 0), collection, criteria, keywords, selected, remove, mode, str(ids), ) facets = cache.get('search_facets_html_%s' % hash) sort = sort.startswith('random') and 'random' or sort.split()[0] sort = sort.endswith('_sort') and sort[:-5] or sort federated_search_query = reduce( reduce_federated_search_query, criteria, keywords) federated_search = sidebar_api_raw( request, federated_search_query, cached_only=True ) if federated_search_query else None return render_to_response('results.html', { 'criteria': map(readable_criteria, criteria), 'query': query, 'keywords': keywords, 'hiddenfields': hiddenfields, 'records': records, 'hits': hits, 'page': page, 'pages': (hits - 1) / pagesize + 1, 'pagesize': pagesize, 'prev_page': prev_page_url, 'next_page': next_page_url, 'reset_url': url, 'form_url': form_url, 'limit_url': limit_url, 'limit_url_orquery': limit_url_orquery, 'facets': facets, 'facets_url': facets_url, 'orfacet': orfacet, 'orquery': orquery, 'sort': sort, 'random': random.random(), 'viewmode': viewmode, 'federated_sources': bool(available_federated_sources(request.user)), 'federated_search': federated_search, 'federated_search_query': federated_search_query, 'pagination_helper': [None] * hits, 'has_record_created_criteria': any( f.startswith('created:') for f in criteria), 'has_last_modified_criteria': any( f.startswith('modified:') for f in criteria), }, context_instance=RequestContext(request))
def data_import_file(request, file): available_collections = filter_by_access(request.user, Collection) writable_collection_ids = list( filter_by_access(request.user, Collection, write=True) .values_list('id', flat=True) ) if not available_collections: raise Http404 available_fieldsets = FieldSet.for_user(request.user) def _get_fields(): return Field.objects.select_related('standard').all() \ .order_by('standard', 'name') def _get_display_label(field): return '%s (%s) [%d]' % (field.label, field.name, field.id) def _field_choices(): fsf = list( FieldSetField.objects.select_related('fieldset', 'field') .exclude(fieldset__name__startswith='browse-collection-') .exclude(fieldset__name='facet-fields') .order_by('fieldset__name', 'order', 'field__label') ) grouped = {} for f in fsf: grouped.setdefault( (f.fieldset.title, f.fieldset.id), []).append(f.field) others = list( (field.id, _get_display_label(field)) for field in Field.objects.exclude( id__in=[f.field.id for f in fsf] ).order_by('label') ) choices = [ ('', '-' * 10) ] + [ (set[0], [(f.id, _get_display_label(f)) for f in fields]) for set, fields in sorted(grouped.iteritems(), key=lambda s: s[0][0]) ] if others: choices.append(('Others', others)) return choices class ImportOptionsForm(forms.Form): separator = forms.CharField(required=False) collections = forms.MultipleChoiceField( choices=( ( c.id, '%s%s' % ( '*' if c.id in writable_collection_ids else '', c.title ) ) for c in available_collections ), widget=forms.CheckboxSelectMultiple ) fieldset = FieldSetChoiceField(user=request.user, default_label='any') update = forms.BooleanField( label='Update existing records', initial=True, required=False) add = forms.BooleanField( label='Add new records', initial=True, required=False) test = forms.BooleanField( label='Test import only', initial=False, required=False) personal = forms.BooleanField( label='Personal records', initial=False, required=False) def clean(self): cleaned_data = self.cleaned_data if any(self.errors): return cleaned_data personal = cleaned_data['personal'] if not personal: for c in map(int, cleaned_data['collections']): if c not in writable_collection_ids: self._errors['collections'] = ErrorList( ["Can only add personal records to " "selected collections"]) del cleaned_data['collections'] return cleaned_data return cleaned_data class MappingForm(forms.Form): fieldname = forms.CharField( widget=DisplayOnlyTextWidget, required=False) mapping = forms.ChoiceField(choices=_field_choices(), required=False) separate = forms.BooleanField(required=False) label = forms.CharField(required=False) hidden = forms.BooleanField(required=False) refinement = forms.CharField(required=False) class BaseMappingFormSet(forms.formsets.BaseFormSet): def clean(self): if any(self.errors): return _dcidentifier = Field.objects.get( name='identifier', standard__prefix='dc') _identifier_ids = list( _dcidentifier.get_equivalent_fields() .values_list('id', flat=True) ) + [_dcidentifier.id] for i in range(self.total_form_count()): m = self.forms[i].cleaned_data['mapping'] if m and int(m) in _identifier_ids: return raise forms.ValidationError, "At least one field must be mapped " \ "to an identifier field." create_mapping_formset = formset_factory( MappingForm, extra=0, formset=BaseMappingFormSet, can_order=True) def analyze(collections=None, separator=None, separate_fields=None, fieldset=None): try: infile = os.path.join( _get_scratch_dir(), _get_filename(request, file) ) with open(infile, 'rU') as csvfile: imp = SpreadsheetImport( csvfile, collections, separator=separator, separate_fields=separate_fields, preferred_fieldset=fieldset ) return imp, imp.analyze() except IOError: raise Http404() if request.method == 'POST': form = ImportOptionsForm(request.POST) mapping_formset = create_mapping_formset(request.POST, prefix='m') if form.is_valid() and mapping_formset.is_valid(): if int(form.cleaned_data.get('fieldset') or 0): fieldsets = available_fieldsets.get( id=form.cleaned_data['fieldset']) else: fieldsets = None imp, preview_rows = analyze( available_collections.filter( id__in=form.cleaned_data['collections']), form.cleaned_data['separator'], dict( (f.cleaned_data['fieldname'], f.cleaned_data['separate']) for f in mapping_formset.forms ), fieldsets ) store_settings( request.user, 'data_import_file_%s' % imp.field_hash, simplejson.dumps( dict( options=form.cleaned_data, mapping=mapping_formset.cleaned_data) ) ) if request.POST.get('import_button'): arg = dict( filename=_get_filename(request, file), separator=form.cleaned_data['separator'], collection_ids=map( int, form.cleaned_data['collections']), update=form.cleaned_data['update'], add=form.cleaned_data['add'], test=form.cleaned_data['test'], personal=form.cleaned_data['personal'], mapping=dict( ( f.cleaned_data['fieldname'], int(f.cleaned_data['mapping']) ) for f in mapping_formset.forms if f.cleaned_data['mapping'] ), separate_fields=dict( ( f.cleaned_data['fieldname'], f.cleaned_data['separate'] ) for f in mapping_formset.forms ), labels=dict( (f.cleaned_data['fieldname'], f.cleaned_data['label']) for f in mapping_formset.forms ), order=dict( ( f.cleaned_data['fieldname'], safe_int(f.cleaned_data['ORDER'], 0) ) for f in mapping_formset.forms ), hidden=dict( (f.cleaned_data['fieldname'], f.cleaned_data['hidden']) for f in mapping_formset.forms ), refinements=dict( ( f.cleaned_data['fieldname'], f.cleaned_data['refinement'] ) for f in mapping_formset.forms ), ) task = csvimport.delay(owner=request.user.id, **arg) messages.add_message( request, messages.INFO, message='Import job has been submitted.' ) return HttpResponseRedirect( "%s?highlight=%s" % (reverse('workers-jobs'), task.id)) else: imp, preview_rows = analyze() else: imp, preview_rows = analyze() # try to load previously stored settings key = 'data_import_file_%s' % imp.field_hash values = load_settings(request.user, key) if key in values: value = simplejson.loads(values[key][0]) mapping = value['mapping'] options = value['options'] else: mapping = [ dict(fieldname=f, mapping=v.id if v else 0, separate=True) for f, v in imp.mapping.iteritems() ] options = None mapping = sorted(mapping, key=lambda m: m['fieldname']) mapping_formset = create_mapping_formset(initial=mapping, prefix='m') form = ImportOptionsForm(initial=options) return render_to_response( 'data_import_file.html', { 'form': form, 'preview_rows': preview_rows, 'mapping_formset': mapping_formset, 'writable_collections': bool(writable_collection_ids), }, context_instance=RequestContext(request) )
def search(request, id=None, name=None, selected=False, json=False): #print "solr search\n\n\n!!!!!!\n\n\n" collection = id and get_object_or_404(filter_by_access(request.user, Collection), id=id) or None """ #print "------------------------------------------------------view.search----------------------------" #print "request" #print request """ if request.method == "POST": update_record_selection(request) # redirect to get request with updated parameters q = request.GET.copy() q.update(request.POST) q = clean_record_selection_vars(q) for i, v in q.items(): if i != 'c': q[i] = v # replace multiple values with last one except for criteria ('c') q.pop('v.x', None) q.pop('v.y', None) q.pop('x', None) q.pop('y', None) return HttpResponseRedirect(request.path + '?' + q.urlencode()) # get parameters relevant for search criteria = request.GET.getlist('c') remove = request.GET.get('rem', None) if remove and remove in criteria: criteria.remove(remove) keywords = request.GET.get('kw', '') #print [ item.encode('ascii') for item in ast.literal_eval(keywords) ] # get parameters relevant for view viewmode = request.GET.get('v', 'thumb') if viewmode == 'list': pagesize = max(min(safe_int(request.GET.get('ps', '50'), 50), 100), 5) else: pagesize = max(min(safe_int(request.GET.get('ps', '30'), 30), 50), 5) page = safe_int(request.GET.get('page', '1'), 1) sort = request.GET.get('s', 'title_sort').lower() if not sort.endswith(" asc") and not sort.endswith(" desc"): sort += " asc" orquery = request.GET.get('or', None) user = request.user if request.GET.has_key('action'): page = safe_int(request.GET.get('op', '1'), 1) if selected: selected = request.session.get('selected_records', ()) (hits, records, search_facets, orfacet, query, fields) = run_search(user, collection, criteria, keywords, sort, page, pagesize, orquery, selected, remove, produce_facets=False) #print "solr.views 414 criteria %s keywords %s hits %s records %s" %(criteria, keywords, hits, records) if json: return (hits, records, viewmode) if collection: url = reverse('solr-search-collection', kwargs={'id': collection.id, 'name': collection.name}) furl = reverse('solr-search-collection-facets', kwargs={'id': collection.id, 'name': collection.name}) elif hasattr(selected, '__len__'): url = reverse('solr-selected') furl = reverse('solr-selected-facets') else: url = reverse('solr-search') furl = reverse('solr-search-facets') q = request.GET.copy() q = clean_record_selection_vars(q) q.pop('or', None) q.pop('rem', None) q.pop('action', None) q.pop('page', None) q.pop('op', None) q.pop('v.x', None) q.pop('v.y', None) q.pop('x', None) q.pop('y', None) q['s'] = q.get('s', sort) q['v'] = q.get('v', 'thumb') q.setlist('c', criteria) hiddenfields = [('op', page)] #for f in q: # if f != 'kw': # for l in q.getlist(f): # hiddenfields.append((f, l)) qurl = q.urlencode() q.setlist('c', filter(lambda c: c != orquery, criteria)) qurl_orquery = q.urlencode() limit_url = "%s?%s%s" % (url, qurl, qurl and '&' or '') limit_url_orquery = "%s?%s%s" % (url, qurl_orquery, qurl_orquery and '&' or '') facets_url = "%s?%s%s" % (furl, qurl, qurl and '&' or '') form_url = "%s?%s" % (url, q.urlencode()) prev_page_url = None next_page_url = None if page > 1: q['page'] = page - 1 prev_page_url = "%s?%s" % (url, q.urlencode()) if page < (hits - 1) / pagesize + 1: q['page'] = page + 1 next_page_url = "%s?%s" % (url, q.urlencode()) def readable_criteria(c): (f, o) = c.split(':', 1) negated = f.startswith('-') f = f[1 if negated else 0:] if search_facets.has_key(f): return dict(facet=c, term=search_facets[f].display_value(o), label=search_facets[f].label, negated=negated, or_available=not negated and search_facets[f].or_available()) else: return dict(facet=c, term=o, label='Unknown criteria', negated=negated, or_available=False) def reduce_federated_search_query(q, c): (f, o) = c.split(':', 1) if f.startswith('-') or not search_facets.has_key(f): # can't negate in federated search return q v = search_facets[f].federated_search_query(o) return v if not q else '%s %s' % (q, v) mode, ids = get_collection_visibility_preferences(user) hash = calculate_hash(getattr(user, 'id', 0), collection, criteria, keywords, selected, remove, mode, str(ids), ) #print hash facets = cache.get('search_facets_html_%s' % hash) sort = sort.startswith('random') and 'random' or sort.split()[0] sort = sort.endswith('_sort') and sort[:-5] or sort federated_search_query = reduce(reduce_federated_search_query, criteria, keywords) federated_search = sidebar_api_raw( request, federated_search_query, cached_only=True) if federated_search_query else None query_string = "" """if len(criteria)>0 : query_string += "search=advance " else : query_string += "search=simple " """ query_list = {} crit_pattern = re.compile("(?P<type>.*?(\_t)?):\"?(?P<value>.*?)\"?$") #crit_pattern = re.compile("(?P<type>.*?\_t):(?P<value>.*?)") for c in criteria : m = crit_pattern.match(c) add_to_dict(query_list,str(m.group('type')),str(m.group('value')).replace(" ",'+').replace("\"",'')) """ if query_list.has_key(str(m.group('type'))) : query_list.update({str(m.group('type')):query_list[str(m.group('type'))]+"%26\""+str(m.group('value')).replace(" ",'+').replace("\"",'')+"\""}) else: query_list.update({str(m.group('type')):"\""+str(m.group('value')).replace(" ",'+').replace("\"",'')+"\""}) """ """ #print "keywords is" #print keywords """ kws_list = re.split('(-?\".*?\")',keywords) while "" in kws_list: kws_list.remove("") while " " in kws_list: kws_list.remove(" ") kws="" kws_not = "" query_terms="" for kw in kws_list: kw = kw.replace("\"","") if kw is '': # empty search pass # nothing to do elif "=" in kw: if not query_terms == "": query_terms += ", " query_terms += kw elif kw[0]=='-': kw = kw.replace('\"','') if kws_not is "": kws_not += kw.replace('-','') else : kws_not += "+"+kw.replace('-','') else: kw = kw.replace('\"','') if kws is "": kws += kw else: kws += "+"+kw #print "kws = " #print kws #print "kws_not =" #print kws_not if kws.startswith("keywords="): kws = kws.replace("keywords=","") while kws.endswith('\\'): kws = kws[:-1] query_string += "keywords=" + kws query_string = query_string.replace('\"','') # query_string += ";"+str(query_list) for key in query_list.keys() : value = query_list[key] for v in value: while v.endswith('\\'): v = v[:-1] q = key+"="+v query_string = query_string+','+q.replace("\"", "").replace('_t','') if not query_terms is '': query_string += ', '+query_terms if not kws_not is '': query_string += ', '+"-="+kws_not #print "\n\n\n\n\n--------------------------Query String is:----s----------------------------" #print query_string+"\n\n\n\n\n" return render_to_response('results.html', {'criteria': map(readable_criteria, criteria), 'query': query_string, 'keywords': kws,#keywords, 'hiddenfields': hiddenfields, 'records': records, 'hits': hits, 'page': page, 'pages': (hits - 1) / pagesize + 1, 'pagesize': pagesize, 'prev_page': prev_page_url, 'next_page': next_page_url, 'reset_url': url, 'form_url': form_url, 'limit_url': limit_url, 'limit_url_orquery': limit_url_orquery, 'facets': facets, 'facets_url': facets_url, 'orfacet': orfacet, 'orquery': orquery, 'sort': sort, 'random': random.random(), 'viewmode': viewmode, 'federated_search': federated_search, 'federated_search_query':query_string, 'pagination_helper': [None] * hits, 'has_record_created_criteria': any(f.startswith('created:') for f in criteria), 'has_last_modified_criteria': any(f.startswith('modified:') for f in criteria), }, context_instance=RequestContext(request))
def search(request, id=None, name=None, selected=False, json=False): #print "solr search\n\n\n!!!!!!\n\n\n" collection = id and get_object_or_404( filter_by_access(request.user, Collection), id=id) or None """ #print "------------------------------------------------------view.search----------------------------" #print "request" #print request """ if request.method == "POST": update_record_selection(request) # redirect to get request with updated parameters q = request.GET.copy() q.update(request.POST) q = clean_record_selection_vars(q) for i, v in q.items(): if i != 'c': q[i] = v # replace multiple values with last one except for criteria ('c') q.pop('v.x', None) q.pop('v.y', None) q.pop('x', None) q.pop('y', None) return HttpResponseRedirect(request.path + '?' + q.urlencode()) # get parameters relevant for search criteria = request.GET.getlist('c') remove = request.GET.get('rem', None) if remove and remove in criteria: criteria.remove(remove) keywords = request.GET.get('kw', '') #print [ item.encode('ascii') for item in ast.literal_eval(keywords) ] # get parameters relevant for view viewmode = request.GET.get('v', 'thumb') if viewmode == 'list': pagesize = max(min(safe_int(request.GET.get('ps', '50'), 50), 100), 5) else: pagesize = max(min(safe_int(request.GET.get('ps', '30'), 30), 50), 5) page = safe_int(request.GET.get('page', '1'), 1) sort = request.GET.get('s', 'title_sort').lower() if not sort.endswith(" asc") and not sort.endswith(" desc"): sort += " asc" orquery = request.GET.get('or', None) user = request.user if request.GET.has_key('action'): page = safe_int(request.GET.get('op', '1'), 1) if selected: selected = request.session.get('selected_records', ()) (hits, records, search_facets, orfacet, query, fields) = run_search(user, collection, criteria, keywords, sort, page, pagesize, orquery, selected, remove, produce_facets=False) #print "solr.views 414 criteria %s keywords %s hits %s records %s" %(criteria, keywords, hits, records) if json: return (hits, records, viewmode) if collection: url = reverse('solr-search-collection', kwargs={ 'id': collection.id, 'name': collection.name }) furl = reverse('solr-search-collection-facets', kwargs={ 'id': collection.id, 'name': collection.name }) elif hasattr(selected, '__len__'): url = reverse('solr-selected') furl = reverse('solr-selected-facets') else: url = reverse('solr-search') furl = reverse('solr-search-facets') q = request.GET.copy() q = clean_record_selection_vars(q) q.pop('or', None) q.pop('rem', None) q.pop('action', None) q.pop('page', None) q.pop('op', None) q.pop('v.x', None) q.pop('v.y', None) q.pop('x', None) q.pop('y', None) q['s'] = q.get('s', sort) q['v'] = q.get('v', 'thumb') q.setlist('c', criteria) hiddenfields = [('op', page)] #for f in q: # if f != 'kw': # for l in q.getlist(f): # hiddenfields.append((f, l)) qurl = q.urlencode() q.setlist('c', filter(lambda c: c != orquery, criteria)) qurl_orquery = q.urlencode() limit_url = "%s?%s%s" % (url, qurl, qurl and '&' or '') limit_url_orquery = "%s?%s%s" % (url, qurl_orquery, qurl_orquery and '&' or '') facets_url = "%s?%s%s" % (furl, qurl, qurl and '&' or '') form_url = "%s?%s" % (url, q.urlencode()) prev_page_url = None next_page_url = None if page > 1: q['page'] = page - 1 prev_page_url = "%s?%s" % (url, q.urlencode()) if page < (hits - 1) / pagesize + 1: q['page'] = page + 1 next_page_url = "%s?%s" % (url, q.urlencode()) def readable_criteria(c): (f, o) = c.split(':', 1) negated = f.startswith('-') f = f[1 if negated else 0:] if search_facets.has_key(f): return dict(facet=c, term=search_facets[f].display_value(o), label=search_facets[f].label, negated=negated, or_available=not negated and search_facets[f].or_available()) else: return dict(facet=c, term=o, label='Unknown criteria', negated=negated, or_available=False) def reduce_federated_search_query(q, c): (f, o) = c.split(':', 1) if f.startswith('-') or not search_facets.has_key(f): # can't negate in federated search return q v = search_facets[f].federated_search_query(o) return v if not q else '%s %s' % (q, v) mode, ids = get_collection_visibility_preferences(user) hash = calculate_hash( getattr(user, 'id', 0), collection, criteria, keywords, selected, remove, mode, str(ids), ) #print hash facets = cache.get('search_facets_html_%s' % hash) sort = sort.startswith('random') and 'random' or sort.split()[0] sort = sort.endswith('_sort') and sort[:-5] or sort federated_search_query = reduce(reduce_federated_search_query, criteria, keywords) federated_search = sidebar_api_raw( request, federated_search_query, cached_only=True) if federated_search_query else None query_string = "" """if len(criteria)>0 : query_string += "search=advance " else : query_string += "search=simple " """ query_list = {} crit_pattern = re.compile("(?P<type>.*?(\_t)?):\"?(?P<value>.*?)\"?$") #crit_pattern = re.compile("(?P<type>.*?\_t):(?P<value>.*?)") for c in criteria: m = crit_pattern.match(c) add_to_dict(query_list, str(m.group('type')), str(m.group('value')).replace(" ", '+').replace("\"", '')) """ if query_list.has_key(str(m.group('type'))) : query_list.update({str(m.group('type')):query_list[str(m.group('type'))]+"%26\""+str(m.group('value')).replace(" ",'+').replace("\"",'')+"\""}) else: query_list.update({str(m.group('type')):"\""+str(m.group('value')).replace(" ",'+').replace("\"",'')+"\""}) """ """ #print "keywords is" #print keywords """ kws_list = re.split('(-?\".*?\")', keywords) while "" in kws_list: kws_list.remove("") while " " in kws_list: kws_list.remove(" ") kws = "" kws_not = "" query_terms = "" for kw in kws_list: kw = kw.replace("\"", "") if kw is '': # empty search pass # nothing to do elif "=" in kw: if not query_terms == "": query_terms += ", " query_terms += kw elif kw[0] == '-': kw = kw.replace('\"', '') if kws_not is "": kws_not += kw.replace('-', '') else: kws_not += "+" + kw.replace('-', '') else: kw = kw.replace('\"', '') if kws is "": kws += kw else: kws += "+" + kw #print "kws = " #print kws #print "kws_not =" #print kws_not if kws.startswith("keywords="): kws = kws.replace("keywords=", "") while kws.endswith('\\'): kws = kws[:-1] query_string += "keywords=" + kws query_string = query_string.replace('\"', '') # query_string += ";"+str(query_list) for key in query_list.keys(): value = query_list[key] for v in value: while v.endswith('\\'): v = v[:-1] q = key + "=" + v query_string = query_string + ',' + q.replace("\"", "").replace( '_t', '') if not query_terms is '': query_string += ', ' + query_terms if not kws_not is '': query_string += ', ' + "-=" + kws_not #print "\n\n\n\n\n--------------------------Query String is:----s----------------------------" #print query_string+"\n\n\n\n\n" return render_to_response( 'results.html', { 'criteria': map(readable_criteria, criteria), 'query': query_string, 'keywords': kws, #keywords, 'hiddenfields': hiddenfields, 'records': records, 'hits': hits, 'page': page, 'pages': (hits - 1) / pagesize + 1, 'pagesize': pagesize, 'prev_page': prev_page_url, 'next_page': next_page_url, 'reset_url': url, 'form_url': form_url, 'limit_url': limit_url, 'limit_url_orquery': limit_url_orquery, 'facets': facets, 'facets_url': facets_url, 'orfacet': orfacet, 'orquery': orquery, 'sort': sort, 'random': random.random(), 'viewmode': viewmode, 'federated_search': federated_search, 'federated_search_query': query_string, 'pagination_helper': [None] * hits, 'has_record_created_criteria': any(f.startswith('created:') for f in criteria), 'has_last_modified_criteria': any(f.startswith('modified:') for f in criteria), }, context_instance=RequestContext(request))