def show_entry(request, rid, template='afg/entry_page.html', api=False):
try:
entry = DiaryEntry.objects.get(report_key=rid)
except DiaryEntry.DoesNotExist:
try:
entry = DiaryEntry.objects.get(id=int(rid))
except (ValueError, DiaryEntry.DoesNotExist):
raise Http404
phrases = Phrase.objects.filter(entry_count__gt=1,
entry_count__lt=10, entries=entry)
# Equivalent query pre-de-normalization:
# phrases = list(Phrase.objects.raw("""
# SELECT sub.* FROM
# (SELECT p.id, p.phrase, COUNT(pe2.diaryentry_id) AS entry_count FROM
# afg_phrase_entries pe2, afg_phrase p
# INNER JOIN afg_phrase_entries pe1 ON pe1.phrase_id = p.id
# WHERE pe1.diaryentry_id=%s AND p.id=pe2.phrase_id
# GROUP BY p.phrase, p.id) AS sub
# WHERE entry_count > 1 AND entry_count < 10;
# """, [entry.id]))
phrase_ids = [p.id for p in phrases]
dest_ids = defaultdict(list)
if phrase_ids:
cursor = connection.cursor()
# Using modulus not params here because we need to do funky literalizing of
# the table
cursor.execute("""
SELECT pe.phrase_id, d.id FROM afg_phrase_entries pe
INNER JOIN afg_diaryentry d ON pe.diaryentry_id=d.id
WHERE pe.phrase_id IN (SELECT * FROM (VALUES %s) AS phrase_id_set);
""" % (",".join("(%s)" % i for i in phrase_ids)))
for row in cursor.fetchall():
dest_ids[int(row[0])].append(row[1])
phrase_entries = [(phrase, dest_ids[phrase.id]) for phrase in phrases]
if api:
return utils.render_json(request, {
'entry': entry.to_dict(),
'phrase_entries': [{
'phrase': p.phrase,
'entry_ids': ids,
} for p, ids in phrase_entries],
})
return utils.render_request(request, template, {
'entry': entry,
'phrase_entries': phrase_entries,
})
def show_entry(request, rid, template='afg/entry_page.html', api=False):
try:
entry = DiaryEntry.objects.get(report_key=rid)
except DiaryEntry.DoesNotExist:
raise Http404
if api:
return utils.render_json(request, {
'entry': entry.to_dict(),
})
return utils.render_request(request, template, {
'entry': entry,
})
def search(request, about=False, api=False):
sqs = SearchQuerySet()
params = {}
# Full text search
q = request.GET.get('q', None)
if q:
sqs = sqs.auto_query(q).highlight()
params['q'] = q
# prepare fields for faceting; dates are special-cased later.
summary = None
date_fields = {}
for facet in DiaryEntryIndex.search_facet_display:
field = DiaryEntryIndex.fields[facet]
if isinstance(field, haystack.fields.DateTimeField):
date_fields[facet] = field
else:
sqs = sqs.facet(facet)
# XXX: Set field-specific range limit for total casualties
sqs = sqs.raw_params(**{'f.total_casualties_exact.facet.limit': 200})
# Narrow query set by given facets
for key,val in request.GET.iteritems():
if val:
# Add an "exact" param and split by '__'. If the field already has
# e.g. __gte, the __exact addendum is ignored, since we only look
# at the first two parts.
field_name, lookup = (key + "__exact").rsplit(r'__')[0:2]
# "type" is a reserved name for Solr, so munge it to "type_"
if field_name == "type":
field_name = "type_"
field = DiaryEntryIndex.fields.get(field_name, None)
if field and field.faceted:
# Dates are handled specially below
if isinstance(field, haystack.fields.DateTimeField):
continue
elif isinstance(field, haystack.fields.IntegerField):
try:
clean_val = int(val)
except ValueError:
continue
elif isinstance(field, haystack.fields.FloatField):
try:
clean_val = float(val)
except ValueError:
continue
else:
clean_val = sqs.query.clean(val)
if lookup == 'exact':
sqs = sqs.narrow(u'%s:"%s"' % (field.index_fieldname + "_exact", clean_val))
elif lookup == 'gte':
sqs = sqs.narrow(u"%s:[%s TO *]" % (field.index_fieldname, clean_val))
elif lookup == 'lte':
sqs = sqs.narrow(u"%s:[* TO %s]" % (field.index_fieldname, clean_val))
else:
continue
params[key] = val
# Narrow query set by given date facets
for key, field in date_fields.iteritems():
start_str = request.GET.get(key + '__gte', '')
end_str = request.GET.get(key + '__lte', '')
start = None
end = None
if start_str:
try:
start = datetime.datetime(*[int(a) for a in (start_str + '-1-1').split('-')[0:3]])
except ValueError:
start = None
if end_str:
try:
end = datetime.datetime(*[int(a) for a in ((end_str + '-1-1').split('-')[0:3])])
except ValueError:
end = None
if not start and not end:
# Legacy (deprecated) date format -- here to preserve old URLs
day = int(request.GET.get(key + '__day', 0))
month = int(request.GET.get(key + '__month', 0))
year = int(request.GET.get(key + '__year', 0))
if year:
if month:
if day:
start = datetime.datetime(year, month, day)
end = start + datetime.timedelta(days=1)
else:
start = datetime.datetime(year, month, 1)
next_month = start + datetime.timedelta(days=31)
end = datetime.datetime(next_month.year, next_month.month, 1)
else:
start = datetime.datetime(year, 1, 1)
end = datetime.datetime(year + 1, 1, 1)
if start:
sqs = sqs.narrow("%s:[%s TO *]" % (key, start.isoformat() + 'Z'))
params[key + '__gte'] = start.strftime("%Y-%m-%d")
else:
start = DiaryEntryIndex.min_date
if end:
sqs = sqs.narrow("%s:[* TO %s]" % (key, end.isoformat() + 'Z'))
params[key + '__lte'] = end.strftime("%Y-%m-%d")
else:
end = DiaryEntryIndex.max_date
span = max(1, (end - start).days)
gap = max(1, int(span / 100)) # target 100 facets
sqs = sqs.date_facet(key, start, end, 'day', gap)
# sorting
sort = request.GET.get('sort', '')
# Legacy sorting
if request.GET.get('sort_by', ''):
sort_by = request.GET.get('sort_by', '')
if sort_by == 'casualties':
sort_by = 'total_casualties'
sort_dir = request.GET.get('sort_dir', 'asc')
direction_indicator = '-' if sort_dir == 'desc' else ''
sort = direction_indicator + sort_by
if sort.strip('-') not in DiaryEntryIndex.fields:
sort = DiaryEntryIndex.offer_to_sort_by[0][1]
sqs = sqs.order_by(sort)
params['sort'] = sort
# Pagination
p = paginator.Paginator(sqs, 10)
try:
page = p.page(int(request.GET.get('p', 1)))
except (ValueError, paginator.InvalidPage, paginator.EmptyPage):
page = p.page(p.num_pages)
# Results Summaries and highlighting
entries = []
for entry in page.object_list:
if entry.highlighted:
excerpt = mark_safe(u"... %s ..." % entry.highlighted['text'][0])
else:
excerpt = (entry.summary or '')[0:200] + "..."
entries.append((entry, excerpt))
# Choices
counts = sqs.facet_counts()
choices = utils.OrderedDict()
for key in DiaryEntryIndex.search_facet_display:
field = DiaryEntryIndex.fields[key]
choice = None
if isinstance(field, haystack.fields.CharField):
facets = sorted((k, k, c) for k, c in counts['fields'][key] if c > 0)
if facets:
choice = {
'choices': facets,
'type': 'text',
'value': params.get(key, ''),
}
elif isinstance(field, haystack.fields.IntegerField):
# Integer choices
facets = sorted([(int(k), c) for k,c in counts['fields'][key] if c > 0])
if facets:
choice = {
'type': 'min_max',
'counts': [c for k,c in facets],
'vals': [k for k,c in facets],
'min_value': facets[0][0],
'max_value': facets[-1][0],
'chosen_min': params.get(key + '__gte', ''),
'chosen_max': params.get(key + '__lte', ''),
}
elif isinstance(field, haystack.fields.DateTimeField):
facets = sorted(counts['dates'].get(key, {}).iteritems())
if facets:
val_counts = []
vals = []
last_dt = None
for d,c in facets:
if c > 0:
try:
last_dt = _iso_to_datetime(d)
val_counts.append(c)
vals.append(last_dt.strftime('%Y-%m-%d'))
except (TypeError, ValueError):
pass
if vals and last_dt:
max_value = min(
_iso_to_datetime(counts['dates'][key]['end']),
DiaryEntryIndex.max_date,
last_dt + datetime.timedelta(
days=int(re.sub('[^\d]', '', counts['dates'][key]['gap']))
)
)
vals.append(max_value.strftime('%Y-%m-%d'))
val_counts.append(0)
choice = {
'type': 'date',
'counts': val_counts,
'vals': vals,
'min_value': vals[0],
'max_value': vals[-1],
'chosen_min': params.get(key + '__gte', ''),
'chosen_max': params.get(key + '__lte', ''),
}
if choice:
choice['title'] = fix_constraint_name(key)
choices[key] = choice
search_url = reverse('afg.search')
# Links to remove constraints
constraints = {}
exclude = set(('sort',))
for key in params.keys():
if key not in exclude:
value = params.pop(key)
constraints[key] = {
'value': value,
'removelink': "%s?%s" % (search_url, urllib.urlencode(params)),
'title': fix_constraint_name(key)
}
params[key] = value
# Links to change sorting
sort_links = []
current_sort = params.pop('sort')
current_key = sort.strip('-')
for display, new_key in DiaryEntryIndex.offer_to_sort_by:
# Change directions only if it's the same sort key
if current_key == new_key:
direction = '' if current_sort[0] == '-' else '-'
else:
direction = '-' if current_sort[0] == '-' else ''
params['sort'] = direction + new_key
sort_links.append({
'link': "%s?%s" % (search_url, urllib.urlencode(params)),
'title': display,
'desc': current_sort[0] == '-',
'current': current_key == new_key,
})
params['sort'] = current_sort
if api:
remapped_choices = {}
for choice, opts in choices.iteritems():
if opts['type'] in ('min_max', 'date'):
remapped_choices[choice] = opts
else:
remapped_choices[choice] = {
'value': opts['value'],
'title': opts['title'],
'choices': [],
}
for disp, val, count in opts['choices']:
if disp or val:
remapped_choices[choice]['choices'].append({
'value': val,
'display': disp,
'count': count,
})
return utils.render_json(request, {
'pagination': {
'p': page.number,
'num_pages': page.paginator.num_pages,
'num_results': page.paginator.count,
},
'entries': [{
'title': e.title,
'date': e.date.isoformat(),
'release': e.release,
'region': e.region,
'report_key': e.report_key,
'excerpt': x,
'total_casualties': e.total_casualties,
} for (e,x) in entries],
'choices': remapped_choices,
'sort': params['sort'],
'params': params,
})
return utils.render_request(request, "afg/search.html", {'page': page,
'about': about,
'entries': entries,
'params': request.GET,
'choices': choices,
'qstring': '%s?%s' % (search_url, urllib.urlencode(params)),
'constraints': constraints,
'sort_links': sort_links,
})
def search(request, about=False, api=False):
params = {}
for key in request.GET:
trans = SEARCH_PARAMS.get(key, None)
if trans and request.GET[key]:
try:
params[trans[0]] = trans[1](request.GET[key])
except ValueError:
continue
# sorting
sort_by = request.GET.get('sort_by', 'date')
sort_dir = request.GET.get('sort_dir', 'asc')
# special handling of full text search
q = params.pop('q', None)
if q:
qs = DiaryEntry.objects.extra(where=['summary_tsv @@ plainto_tsquery(%s)'], params=[q])
else:
qs = DiaryEntry.objects.all()
qs = qs.filter(**params)
direction_indicator = '-' if sort_dir == 'desc' else ''
if sort_by in ('date', 'total_casualties'):
qs = qs.order_by(direction_indicator + sort_by)
# Restore params now that we've finished filtering on the non-model elements
if q:
params['q'] = q
params['sort_by'] = sort_by
params['sort_dir'] = sort_dir
p = paginator.Paginator(qs, 10)
try:
page = p.page(int(request.GET.get('p', 1)))
except (ValueError, paginator.InvalidPage, paginator.EmptyPage):
page = p.page(p.num_pages)
if q:
needles = q.split()
else:
needles = None
entries = [(entry, _excerpt(entry.summary, needles)) for entry in page.object_list]
choices = utils.OrderedDict()
# Date choices
choices['date__year'] = {
'title': 'Year',
'choices': [(d.year, d.year) for d in qs.dates('date', 'year')],
'value': params.get('date__year', ''),
}
if 'date__year' in params or 'date__month' in params:
choices['date__month'] = {
'title': 'Month',
'choices': sorted(set((d.strftime("%B"), d.month) for d in qs.dates('date', 'month'))),
'value': params.get('date__month', ''),
}
if ('date__year' in params and 'date__month' in params) or 'date__day' in params:
choices['date__day'] = {
'title': 'Day',
'choices': [(d.day, d.day) for d in qs.dates('date', 'day')],
'value': params.get('date__day', ''),
}
# General field choices
for field in ('type', 'region', 'attack_on', 'type_of_unit',
'affiliation', 'dcolor', 'classification', 'category'):
cs = list(q.values()[0] for q in qs.distinct().order_by(field).values(field))
choices[field] = {
'title': field.replace('_', ' ').title(),
'choices': zip(cs, cs),
'value': params.get(field, ''),
}
min_max_choices = utils.OrderedDict()
# Integer choices
for field in ('total_casualties', 'civilian_kia', 'civilian_wia', 'host_nation_kia', 'host_nation_wia', 'friendly_kia', 'friendly_wia', 'enemy_kia', 'enemy_wia', 'enemy_detained'):
try:
minimum = qs.order_by(field).values(field)[0][field]
maximum = qs.order_by("-%s" % field).values(field)[0][field]
except IndexError:
continue
if minimum == maximum and \
not params.get(field + '__gte') and \
not params.get(field + '__lte'):
continue
min_max_choices[field] = {
'min': minimum,
'max': maximum,
'title': fix_constraint_name(field),
'min_value': params.get(field + '__gte', ''),
'max_value': params.get(field + '__lte', ''),
}
search_url = reverse('afg.search')
# Links to remove constraints
constraints = {}
exclude = set(('sort_by', 'sort_dir'))
for field in params.keys():
if field not in exclude:
value = params.pop(field)
constraints[field] = {
'value': value,
'removelink': "%s?%s" % (search_url, urllib.urlencode(params)),
'title': fix_constraint_name(field)
}
params[field] = value
# Links to change sorting
sort = {}
for by in ('date', 'total_casualties'):
sort_by = params.pop('sort_by', 'date')
sort_dir = params.pop('sort_dir', 'asc')
params['sort_by'] = by
if sort_by == by:
if sort_dir == 'asc':
params['sort_dir'] = 'desc'
sort[by + '_asc'] = True
else:
sort[by + '_desc'] = True
else:
params['sort_dir'] = sort_dir
sort[by] = "%s?%s" % (search_url, urllib.urlencode(params))
params['sort_by'] = sort_by
params['sort_dir'] = sort_dir
if api:
remapped_choices = {}
for choice, opts in choices.iteritems():
remapped_choices[choice] = {
'value': opts['value'],
'title': opts['title'],
'choices': []
}
for disp, val in opts['choices']:
if disp or val:
remapped_choices[choice]['choices'].append({
'value': val,
'display': disp,
})
return utils.render_json(request, {
'pageination': {
'p': page.number,
'num_pages': page.paginator.num_pages,
'num_results': page.paginator.count,
},
'entries': [{
'report_key': e.report_key,
'excerpt': x
} for (e,x) in entries],
'choices': remapped_choices,
'min_max_choices': min_max_choices,
'sort': {
'sort_by': params['sort_by'],
'sort_dir': params['sort_dir'],
},
'params': params,
})
return utils.render_request(request, "afg/search.html", {'page': page,
'about': about,
'entries': entries,
'params': request.GET,
'choices': choices,
'min_max_choices': min_max_choices,
'qstring': '%s?%s' % (search_url, urllib.urlencode(params)),
'constraints': constraints,
'sort': sort,
})
