def _parse_facets(qd): """ Given a request, returns a quadruple of: * A SearchQuerySet constrained by the request.GET vars. * a dict of constraints that have been applied * a list of potential choices for new constraints * a querystring that produces these constraints (without pagination or sorting) """ sqs = SearchQuerySet() constraints = [] # Process constraints, and execute facets for name, field in settings.ALL_FIELDS.iteritems(): if field['document']: terms = qd.get(field['name'], qd.get('document_', None)) if terms: sqs = sqs.auto_query(terms).highlight() constraints.append({ 'value': terms, 'key': name, 'type': 'document', 'display': field['display_name'], }) elif field['faceted']: sqs = sqs.raw_params(**{ 'f.%s_exact.facet.limit' % field['name']: field['facet_limit'] }) if field['type'] == 'date': sqs, start, end = _narrow_range(sqs, field, qd) if start: constraints.append({ 'value': start.strftime("%Y-%m-%d"), 'key': name + '__gte', 'type': 'date', 'display': "%s - more than" % field['display_name'], }) if end: constraints.append({ 'value': end.strftime("%Y-%m-%d"), 'key': name + '__lte', 'type': 'date', 'display': "%s - less than" % field['display_name'], }) if start or end or name in settings.FACET_DISPLAY: if not end: end = utils.d_to_dt(getattr(SearchQuerySet().order_by("-" + name)[0], name)) if not start: start = utils.d_to_dt(getattr(SearchQuerySet().order_by(name)[0], name)) span = max(1, (end - start).days) gap = max(1, int(span / 100)) sqs = sqs.date_facet(name, start, end, 'day', gap) elif field['type'] in ('int', 'float', 'latitude', 'longitude'): sqs, start, end = _narrow_range(sqs, field, qd) if start: constraints.append({ 'value': start, 'key': name + '__gte', 'type': 'min_max', 'display': "%s - more than" % field['display_name'], }) if end: constraints.append({ 'value': end, 'key': name + '__lte', 'type': 'min_max', 'display': "%s - less than" % field['display_name'], }) if start or end or name in settings.FACET_DISPLAY: sqs = sqs.facet(name) # if not end: # end = getattr(SearchQuerySet().order_by("-" + name)[0], name) # if not start: # start = getattr(SearchQuerySet().order_by(name)[0], name) # span = max(1, (end - start)) # gap = max(1, int(span / 100)) # exact = "%s_exact" % name # sqs = sqs.raw_params(**{ # "facet.range": exact, # "f.%s.facet.range.start" % exact: start, # "f.%s.facet.range.end" % exact: end, # "f.%s.facet.range.gap" % exact: gap, # }) else: val = qd.get(name, None) if val is not None: constraints.append({ 'value': val, 'key': name, 'type': 'text', 'display': field['display_name'], }) sqs = sqs.narrow('%s:"%s"' % (name + "_exact", val)) if val is not None or name in settings.FACET_DISPLAY: sqs = sqs.facet(name) # Constraint removal links queryargs = dict((p['key'], p['value']) for p in constraints) querystr = urllib.urlencode(queryargs) for param in constraints: # remove key value = queryargs.pop(param['key']) remainder = urllib.urlencode(queryargs) param.update({'remove_link': remainder}) # restore key queryargs[param['key']] = value # Choices counts = sqs.facet_counts() choices = [] for name in settings.FACET_DISPLAY: field = settings.ALL_FIELDS[name] choice = {'field': field} if field['document']: choice.update({ 'type': 'document', 'value': queryargs.get(name, ''), }) elif field['type'] in ('text', 'char'): facets = sorted((k, c) for k, c in counts['fields'][name] if c > 0) if facets: choice.update({ 'choices': facets, 'type': 'text', 'value': queryargs.get(name, '') }) elif field['type'] in ('int', 'float', 'latitude', 'longitude'): facets = sorted([(int(k), c) for k,c in counts['fields'][name] if c > 0]) if facets: choice.update({ 'type': 'min_max', 'counts': [c for k,c in facets], 'vals': [k for k,c in facets], 'min_value': facets[0][0], 'max_value': facets[-1][0], 'chosen_min': queryargs.get(name + '__gte', ''), 'chosen_max': queryargs.get(name + '__lte', ''), }) elif field['type'] == 'date': facets = sorted(counts['dates'].get(name, {}).iteritems()) if facets: val_counts = [] vals = [] last_dt = None for d,c in facets: if c > 0: try: last_dt = utils.iso_to_datetime(d) val_counts.append(c) vals.append(last_dt.strftime('%Y-%m-%d')) except (TypeError, ValueError): pass if vals and last_dt: max_value = min( utils.iso_to_datetime(counts['dates'][name]['end']), utils.d_to_dt( getattr(SearchQuerySet().order_by('-' + name)[0], name) ) + datetime.timedelta(days=1), last_dt + datetime.timedelta( days=int(re.sub('[^\d]', '', counts['dates'][name]['gap'])) ), ) vals.append(max_value.strftime('%Y-%m-%d')) val_counts.append(0) choice.update({ 'type': 'date', 'counts': val_counts, 'vals': vals, 'min_value': vals[0], 'max_value': vals[-1], 'chosen_min': queryargs.get(name + '__gte', ''), 'chosen_max': queryargs.get(name + '__lte', ''), }) choices.append(choice) return sqs, choices, constraints, querystr
def nullable_lambda(func): def nullable(val): return None if val == "<null value>" else func(val) return nullable CONVERSIONS = { 'string': string_conv, 'text': string_conv, 'int': nullable_lambda(int), 'float': nullable_lambda(float), 'latitude': nullable_lambda(float), 'longitude': nullable_lambda(float), 'boolean': lambda b: True if b else False, 'null_boolean': lambda b: None if b == "<null value>" else False if b == False else True, 'date': lambda d: str(iso_to_datetime(d)), } class Command(BaseCommand): args = '<csv file>' help = """Import a structured document set as rows in a CSV file.""" @transaction.commit_manually def handle(self, *args, **kwargs): if len(args) != 1: print("""Usage: python manage.py import_documents <csv file> """) sys.exit(1)