def get_quote(self):
quote_timestamp = self.request.redis_conn.srandmember(
"mission:%s:homepage_quotes" % self.request.mission.name,
)
if quote_timestamp:
return LogLine(
self.request.redis_conn,
self.request.mission.main_transcript,
int(timestamp_to_seconds(quote_timestamp)),
)
def get_quote(self):
quote_timestamp = self.request.redis_conn.srandmember(
"mission:%s:homepage_quotes" % self.request.mission.name,
)
if quote_timestamp:
if '/' in quote_timestamp:
transcript, timestamp = quote_timestamp.rsplit('/', 1)
transcript = "%s/%s" % (self.request.mission.name, transcript)
else:
transcript = self.request.mission.main_transcript
timestamp = quote_timestamp
return LogLine(
self.request.redis_conn,
transcript,
int(timestamp_to_seconds(timestamp)),
)
def get_context_data(self):
# Get the query text
q = self.request.GET.get('q', '')
# Get the offset value
try:
offset = int(
self.request.GET.get('offset', '0')
)
if offset < 0:
offset = 0
except ValueError:
offset = 0
# Is it a special search?
special_value = self.request.redis_conn.get("special_search:%s:%s" % (
self.request.mission.name,
q,
))
if special_value:
self.template_name = "search/special.html"
return {
"q": q,
"text": special_value,
}
# Get the results from Xapian
db = xappy.SearchConnection(
os.path.join(
settings.SITE_ROOT,
'..',
"xappydb",
),
)
query = db.query_parse(
q,
default_op=db.OP_OR,
deny = [ "mission" ],
)
query=db.query_filter(
query,
db.query_composite(db.OP_AND, [
db.query_field("mission", self.request.mission.name),
db.query_field("transcript", self.request.mission.main_transcript),
])
)
results = db.search(
query=query,
startrank=offset,
endrank=offset+PAGESIZE,
checkatleast=-1, # everything (entire xapian db fits in memory, so this should be fine)
sortby="-weight",
)
# Go through the results, building a list of LogLine objects
redis_conn = self.request.redis_conn
log_lines = []
for result in results:
transcript_name, timestamp = result.id.split(":", 1)
log_line = LogLine(redis_conn, transcript_name, int(timestamp))
log_line.speaker = Character(redis_conn, transcript_name.split('/')[0], result.data['speaker_identifier'][0])
log_line.title = mark_safe(result.summarise("text", maxlen=50, ellipsis='…', strict_length=True, hl=None))
log_line.summary = mark_safe(result.summarise("text", maxlen=600, ellipsis='…', hl=('<mark>', '</mark>')))
log_lines.append(log_line)
def page_url(offset):
return reverse("search") + '?' + urllib.urlencode({
'q': q.encode('utf-8'),
'offset': offset,
})
if offset==0:
previous_page = False
else:
previous_page = page_url(offset - PAGESIZE)
if offset+PAGESIZE > results.matches_estimated:
next_page = False
else:
next_page = page_url(offset + PAGESIZE)
thispage = offset / PAGESIZE
maxpage = results.matches_estimated / PAGESIZE
pages_to_show = set([0]) | set([thispage-1, thispage, thispage+1]) | set([maxpage])
if 0 == thispage:
pages_to_show.remove(thispage-1)
if maxpage == thispage:
pages_to_show.remove(thispage+1)
pages = []
class Page(object):
def __init__(self, number, url, selected=False):
self.number = number
self.url = url
self.selected = selected
pages_in_order = list(pages_to_show)
pages_in_order.sort()
for page in pages_in_order:
if len(pages)>0 and page != pages[-1].number:
pages.append('...')
pages.append(Page(page+1, page_url(page*PAGESIZE), page==thispage))
error_info = self.request.redis_conn.hgetall(
"error_page:%s:%s" % (
self.request.mission.name,
'no_search_results',
),
)
if not error_info:
error_info = {}
if error_info.has_key('classic_moment_quote'):
error_quote = LogLine(
self.request.redis_conn,
self.request.mission.main_transcript,
timestamp_to_seconds(error_info['classic_moment_quote'])
)
else:
error_quote = None
return {
'log_lines': log_lines,
'result': results,
'q': q,
'previous_page': previous_page,
'next_page': next_page,
'pages': pages,
'debug': {
'query': query,
},
'error': {
'info': error_info,
'quote': error_quote,
}
}
def get_context_data(self):
# Get the query text
q = self.request.GET.get("q", "")
# Get the offset value
try:
offset = int(self.request.GET.get("offset", "0"))
if offset < 0:
offset = 0
except ValueError:
offset = 0
# Is it a special search?
redis_conn = redis.Redis()
special_value = redis_conn.get("special_search:%s" % q)
if special_value:
self.template_name = "search/special.html"
return {"q": q, "text": special_value}
# Get the results from Xapian
db = xappy.SearchConnection(os.path.join(settings.SITE_ROOT, "..", "xappydb"))
query = db.query_parse(q, default_op=db.OP_OR, deny=["mission"])
# query=db.query_filter(
# query,
# db.query_field("mission", self.request.mission.name),
# )
results = db.search(
query=query, startrank=offset, endrank=offset + PAGESIZE, checkatleast=offset + PAGESIZE + 1
)
# Go through the results, building a list of LogLine objects
log_lines = []
for result in results:
transcript_name, timestamp = result.id.split(":", 1)
log_line = LogLine(redis_conn, transcript_name, int(timestamp))
log_line.speaker = Character(redis_conn, transcript_name.split("/")[0], result.data["speaker"][0])
log_line.title = mark_safe(
result.summarise("text", maxlen=50, ellipsis="…", strict_length=True, hl=None)
)
log_line.summary = mark_safe(
result.summarise("text", maxlen=600, ellipsis="…", hl=("<mark>", "</mark>"))
)
log_lines.append(log_line)
def page_url(offset):
return reverse("search") + "?" + urllib.urlencode({"q": q, "offset": offset})
if offset == 0:
previous_page = False
else:
previous_page = page_url(offset - PAGESIZE)
if offset + PAGESIZE > results.matches_estimated:
next_page = False
else:
next_page = page_url(offset + PAGESIZE)
thispage = offset / PAGESIZE
maxpage = results.matches_estimated / PAGESIZE
pages_to_show = set([0]) | set([thispage - 1, thispage, thispage + 1]) | set([maxpage])
if 0 == thispage:
pages_to_show.remove(thispage - 1)
if maxpage == thispage:
pages_to_show.remove(thispage + 1)
pages = []
class Page(object):
def __init__(self, number, url, selected=False):
self.number = number
self.url = url
self.selected = selected
pages_in_order = list(pages_to_show)
pages_in_order.sort()
for page in pages_in_order:
if len(pages) > 0 and page != pages[-1].number:
pages.append("...")
pages.append(Page(page + 1, page_url(page * PAGESIZE), page == thispage))
return {
"log_lines": log_lines,
"result": results,
"q": q,
"previous_page": previous_page,
"next_page": next_page,
"pages": pages,
"debug": {"query": query},
}
def build_mission(self, mission):
print "Building data visualisations for %s..." % mission.name
for act in list(Act.Query(self.redis_conn, mission.name)):
print ' ... %s' % act.title
# Split the act into sections, one for each bar on the graph
act_duration = act.end - act.start
section_duration = act_duration // 92
# Count the number of log lines in each segment
# and find the maximum number of log lines in a segment
t = act.start
segment_line_counts = []
max_line_count = 0
real_output_path = self.image_output_path % mission.name
while t < act.end:
# Load log lines for this segment
query = LogLine.Query(self.redis_conn,
mission.name).transcript(
mission.main_transcript).range(
t, t + section_duration)
line_count = len(list(query))
# Store segment stats
max_line_count = max(line_count, max_line_count)
segment_line_counts.append(
(t, t + section_duration, line_count))
t += section_duration
# Make sure we have an output directory and work out where to
# write the image
try:
os.makedirs(real_output_path)
except OSError:
pass
graph_file = 'graph_%s_%s.png' % (mission.name, act.number)
output_path = '%s/%s' % (real_output_path, graph_file)
# Add initial draw command
draw_commands = [
'convert',
'-size',
'%dx%d' % (self.width, self.height),
'xc:transparent',
'-fill',
self.end_marker_colour,
'-draw',
"path 'M 1,1 L 10,1 L 5,8 L 1,1",
'-draw',
"path 'M 890,1 L 900,1 L 895,8 L 890,1",
'-fill',
self.graph_bar_colour,
]
# Add initial image map tags
image_map_id = '%s_%s_frequency_graph' % (mission.name, act.number)
image_map = [
'<map id="%s" name="%s">' % (image_map_id, image_map_id)
]
# Iterate over the segments and add them to the draw commands and image map
for i, line in enumerate(segment_line_counts):
start, end, count = line
height = int(
round(count / float(max(max_line_count, 1)) *
self.max_bar_height))
bar_width = 6
bar_spacing = 4
top_left_x = i * (bar_width + bar_spacing) + 2
top_left_y = self.max_bar_height - height + 14
bottom_right_x = top_left_x + bar_width
bottom_right_y = self.max_bar_height + 14
draw_commands.append('-draw')
draw_commands.append(
'rectangle %s,%s,%s,%s' %
(top_left_x, top_left_y, bottom_right_x, bottom_right_y))
if height > 0:
image_map.append(
'<area shape="rect" coords="%(coords)s" href="%(url)s" alt="%(alt)s">'
% {
"url":
'/%s/%s/#show-selection' %
(seconds_to_timestamp(start),
seconds_to_timestamp(end)),
"alt":
'%d lines between %s and %s' %
(count, seconds_to_timestamp(start),
seconds_to_timestamp(end)),
"coords":
'%s,%s,%s,%s' % (top_left_x, top_left_y,
bottom_right_x, bottom_right_y),
})
# Output the basic graph image
draw_commands.append(output_path)
subprocess.call(draw_commands)
# Iterate over the key scenes adding them to the graph and image map
for i, key_scene in enumerate(act.key_scenes()):
print ' - %s' % key_scene.title
top_left_x = int(
(self.graph_background_width / float(act_duration)) *
(key_scene.start - act.start)) + 2
top_left_y = self.max_bar_height + 5 + 14
bottom_right_x = top_left_x + 20
bottom_right_y = top_left_y + 20
marker_image = self.key_scene_marker_files % (i + 1)
subprocess.call([
'composite',
'-geometry',
'+%s+%s' % (top_left_x, top_left_y),
marker_image,
output_path,
output_path,
])
image_map.append(
'<area shape="rect" coords="%(coords)s" href="%(url)s" alt="%(alt)s">'
% {
"url":
'/%s/%s/#show-selection' %
(seconds_to_timestamp(key_scene.start),
seconds_to_timestamp(key_scene.end)),
"alt":
key_scene.title.decode('utf-8'),
"coords":
'%s,%s,%s,%s' % (top_left_x, top_left_y,
bottom_right_x, bottom_right_y),
})
# Finalise the image map
image_map.append('</map>')
self.redis_conn.hmset(
'act:%s:%s:stats' % (mission.name, act.number), {
"image_map": "\n".join(image_map),
"image_map_id": image_map_id,
})
def log_line_query(self):
return LogLine.Query(self.request.redis_conn,
self.request.mission.name)
def get_context_data(self):
# Get the query text
q = self.request.GET.get('q', '')
# Get the offset value
try:
offset = int(self.request.GET.get('offset', '0'))
if offset < 0:
offset = 0
except ValueError:
offset = 0
# Is it a special search?
special_value = self.request.redis_conn.get("special_search:%s:%s" % (
self.request.mission.name,
q,
))
if special_value:
self.template_name = "search/special.html"
return {
"q": q,
"text": special_value,
}
# Get the results from Xapian
db = xappy.SearchConnection(
os.path.join(
settings.SITE_ROOT,
'..',
"xappydb",
), )
query = db.query_parse(
q,
default_op=db.OP_OR,
deny=["mission"],
)
query = db.query_filter(
query,
db.query_composite(db.OP_AND, [
db.query_field("mission", self.request.mission.name),
db.query_field("transcript",
self.request.mission.main_transcript),
]))
results = db.search(
query=query,
startrank=offset,
endrank=offset + PAGESIZE,
checkatleast=
-1, # everything (entire xapian db fits in memory, so this should be fine)
sortby="-weight",
)
# Go through the results, building a list of LogLine objects
redis_conn = self.request.redis_conn
log_lines = []
for result in results:
transcript_name, timestamp = result.id.split(":", 1)
log_line = LogLine(redis_conn, transcript_name, int(timestamp))
log_line.speaker = Character(redis_conn,
transcript_name.split('/')[0],
result.data['speaker_identifier'][0])
log_line.title = mark_safe(
result.summarise("text",
maxlen=50,
ellipsis='…',
strict_length=True,
hl=None))
log_line.summary = mark_safe(
result.summarise("text",
maxlen=600,
ellipsis='…',
hl=('<mark>', '</mark>')))
log_lines.append(log_line)
def page_url(offset):
return reverse("search") + '?' + urllib.urlencode(
{
'q': q.encode('utf-8'),
'offset': offset,
})
if offset == 0:
previous_page = False
else:
previous_page = page_url(offset - PAGESIZE)
if offset + PAGESIZE > results.matches_estimated:
next_page = False
else:
next_page = page_url(offset + PAGESIZE)
thispage = offset / PAGESIZE
maxpage = results.matches_estimated / PAGESIZE
pages_to_show = set([0]) | set([thispage - 1, thispage, thispage + 1
]) | set([maxpage])
if 0 == thispage:
pages_to_show.remove(thispage - 1)
if maxpage == thispage:
pages_to_show.remove(thispage + 1)
pages = []
class Page(object):
def __init__(self, number, url, selected=False):
self.number = number
self.url = url
self.selected = selected
pages_in_order = list(pages_to_show)
pages_in_order.sort()
for page in pages_in_order:
if len(pages) > 0 and page != pages[-1].number:
pages.append('...')
pages.append(
Page(page + 1, page_url(page * PAGESIZE), page == thispage))
error_info = self.request.redis_conn.hgetall(
"error_page:%s:%s" % (
self.request.mission.name,
'no_search_results',
), )
if not error_info:
error_info = {}
if error_info.has_key('classic_moment_quote'):
error_quote = LogLine(
self.request.redis_conn, self.request.mission.main_transcript,
timestamp_to_seconds(error_info['classic_moment_quote']))
else:
error_quote = None
return {
'log_lines': log_lines,
'result': results,
'q': q,
'previous_page': previous_page,
'next_page': next_page,
'pages': pages,
'debug': {
'query': query,
},
'error': {
'info': error_info,
'quote': error_quote,
}
}
def get_context_data(self):
# Get the query text
q = self.request.GET.get("q", "")
# Get the offset value
try:
offset = int(self.request.GET.get("offset", "0"))
if offset < 0:
offset = 0
except ValueError:
offset = 0
# Is it a special search?
special_value = self.request.redis_conn.get("special_search:%s:%s" % (self.request.mission.name, q))
if special_value:
self.template_name = "search/special.html"
return {"q": q, "text": special_value}
# Get the results from Xapian
db = xappy.SearchConnection(os.path.join(settings.SITE_ROOT, "..", "xappydb"))
query = db.query_parse(q, default_op=db.OP_OR, deny=["mission"])
query = db.query_filter(
query,
db.query_composite(
db.OP_AND,
[
db.query_field("mission", self.request.mission.name),
db.query_field("transcript", "%s/TEC" % self.request.mission.name),
],
),
)
results = db.search(
query=query,
startrank=offset,
endrank=offset + PAGESIZE,
checkatleast=-1, # everything (entire xapian db fits in memory, so this should be fine)
sortby="-weight",
)
# Go through the results, building a list of LogLine objects
redis_conn = self.request.redis_conn
log_lines = []
for result in results:
transcript_name, timestamp = result.id.split(":", 1)
log_line = LogLine(redis_conn, transcript_name, int(timestamp))
log_line.speaker = Character(
redis_conn, transcript_name.split("/")[0], result.data["speaker_identifier"][0]
)
log_line.title = mark_safe(
result.summarise("text", maxlen=50, ellipsis="…", strict_length=True, hl=None)
)
log_line.summary = mark_safe(
result.summarise("text", maxlen=600, ellipsis="…", hl=("<mark>", "</mark>"))
)
log_lines.append(log_line)
def page_url(offset):
return reverse("search") + "?" + urllib.urlencode({"q": q, "offset": offset})
if offset == 0:
previous_page = False
else:
previous_page = page_url(offset - PAGESIZE)
if offset + PAGESIZE > results.matches_estimated:
next_page = False
else:
next_page = page_url(offset + PAGESIZE)
thispage = offset / PAGESIZE
maxpage = results.matches_estimated / PAGESIZE
pages_to_show = set([0]) | set([thispage - 1, thispage, thispage + 1]) | set([maxpage])
if 0 == thispage:
pages_to_show.remove(thispage - 1)
if maxpage == thispage:
pages_to_show.remove(thispage + 1)
pages = []
class Page(object):
def __init__(self, number, url, selected=False):
self.number = number
self.url = url
self.selected = selected
pages_in_order = list(pages_to_show)
pages_in_order.sort()
for page in pages_in_order:
if len(pages) > 0 and page != pages[-1].number:
pages.append("...")
pages.append(Page(page + 1, page_url(page * PAGESIZE), page == thispage))
error_info = self.request.redis_conn.hgetall(
"error_page:%s:%s" % (self.request.mission.name, "no_search_results")
)
if not error_info:
error_info = {}
if error_info.has_key("classic_moment_quote"):
error_quote = LogLine(
self.request.redis_conn,
self.request.mission.main_transcript,
timestamp_to_seconds(error_info["classic_moment_quote"]),
)
else:
error_quote = None
return {
"log_lines": log_lines,
"result": results,
"q": q,
"previous_page": previous_page,
"next_page": next_page,
"pages": pages,
"debug": {"query": query},
"error": {"info": error_info, "quote": error_quote},
}
