def get_quote(self): quote_timestamp = self.request.redis_conn.srandmember( "mission:%s:homepage_quotes" % self.request.mission.name, ) if quote_timestamp: return LogLine( self.request.redis_conn, self.request.mission.main_transcript, int(timestamp_to_seconds(quote_timestamp)), )
def get_quote(self): quote_timestamp = self.request.redis_conn.srandmember( "mission:%s:homepage_quotes" % self.request.mission.name, ) if quote_timestamp: if '/' in quote_timestamp: transcript, timestamp = quote_timestamp.rsplit('/', 1) transcript = "%s/%s" % (self.request.mission.name, transcript) else: transcript = self.request.mission.main_transcript timestamp = quote_timestamp return LogLine( self.request.redis_conn, transcript, int(timestamp_to_seconds(timestamp)), )
def get_context_data(self): # Get the query text q = self.request.GET.get('q', '') # Get the offset value try: offset = int( self.request.GET.get('offset', '0') ) if offset < 0: offset = 0 except ValueError: offset = 0 # Is it a special search? special_value = self.request.redis_conn.get("special_search:%s:%s" % ( self.request.mission.name, q, )) if special_value: self.template_name = "search/special.html" return { "q": q, "text": special_value, } # Get the results from Xapian db = xappy.SearchConnection( os.path.join( settings.SITE_ROOT, '..', "xappydb", ), ) query = db.query_parse( q, default_op=db.OP_OR, deny = [ "mission" ], ) query=db.query_filter( query, db.query_composite(db.OP_AND, [ db.query_field("mission", self.request.mission.name), db.query_field("transcript", self.request.mission.main_transcript), ]) ) results = db.search( query=query, startrank=offset, endrank=offset+PAGESIZE, checkatleast=-1, # everything (entire xapian db fits in memory, so this should be fine) sortby="-weight", ) # Go through the results, building a list of LogLine objects redis_conn = self.request.redis_conn log_lines = [] for result in results: transcript_name, timestamp = result.id.split(":", 1) log_line = LogLine(redis_conn, transcript_name, int(timestamp)) log_line.speaker = Character(redis_conn, transcript_name.split('/')[0], result.data['speaker_identifier'][0]) log_line.title = mark_safe(result.summarise("text", maxlen=50, ellipsis='…', strict_length=True, hl=None)) log_line.summary = mark_safe(result.summarise("text", maxlen=600, ellipsis='…', hl=('<mark>', '</mark>'))) log_lines.append(log_line) def page_url(offset): return reverse("search") + '?' + urllib.urlencode({ 'q': q.encode('utf-8'), 'offset': offset, }) if offset==0: previous_page = False else: previous_page = page_url(offset - PAGESIZE) if offset+PAGESIZE > results.matches_estimated: next_page = False else: next_page = page_url(offset + PAGESIZE) thispage = offset / PAGESIZE maxpage = results.matches_estimated / PAGESIZE pages_to_show = set([0]) | set([thispage-1, thispage, thispage+1]) | set([maxpage]) if 0 == thispage: pages_to_show.remove(thispage-1) if maxpage == thispage: pages_to_show.remove(thispage+1) pages = [] class Page(object): def __init__(self, number, url, selected=False): self.number = number self.url = url self.selected = selected pages_in_order = list(pages_to_show) pages_in_order.sort() for page in pages_in_order: if len(pages)>0 and page != pages[-1].number: pages.append('...') pages.append(Page(page+1, page_url(page*PAGESIZE), page==thispage)) error_info = self.request.redis_conn.hgetall( "error_page:%s:%s" % ( self.request.mission.name, 'no_search_results', ), ) if not error_info: error_info = {} if error_info.has_key('classic_moment_quote'): error_quote = LogLine( self.request.redis_conn, self.request.mission.main_transcript, timestamp_to_seconds(error_info['classic_moment_quote']) ) else: error_quote = None return { 'log_lines': log_lines, 'result': results, 'q': q, 'previous_page': previous_page, 'next_page': next_page, 'pages': pages, 'debug': { 'query': query, }, 'error': { 'info': error_info, 'quote': error_quote, } }
def get_context_data(self): # Get the query text q = self.request.GET.get("q", "") # Get the offset value try: offset = int(self.request.GET.get("offset", "0")) if offset < 0: offset = 0 except ValueError: offset = 0 # Is it a special search? redis_conn = redis.Redis() special_value = redis_conn.get("special_search:%s" % q) if special_value: self.template_name = "search/special.html" return {"q": q, "text": special_value} # Get the results from Xapian db = xappy.SearchConnection(os.path.join(settings.SITE_ROOT, "..", "xappydb")) query = db.query_parse(q, default_op=db.OP_OR, deny=["mission"]) # query=db.query_filter( # query, # db.query_field("mission", self.request.mission.name), # ) results = db.search( query=query, startrank=offset, endrank=offset + PAGESIZE, checkatleast=offset + PAGESIZE + 1 ) # Go through the results, building a list of LogLine objects log_lines = [] for result in results: transcript_name, timestamp = result.id.split(":", 1) log_line = LogLine(redis_conn, transcript_name, int(timestamp)) log_line.speaker = Character(redis_conn, transcript_name.split("/")[0], result.data["speaker"][0]) log_line.title = mark_safe( result.summarise("text", maxlen=50, ellipsis="…", strict_length=True, hl=None) ) log_line.summary = mark_safe( result.summarise("text", maxlen=600, ellipsis="…", hl=("<mark>", "</mark>")) ) log_lines.append(log_line) def page_url(offset): return reverse("search") + "?" + urllib.urlencode({"q": q, "offset": offset}) if offset == 0: previous_page = False else: previous_page = page_url(offset - PAGESIZE) if offset + PAGESIZE > results.matches_estimated: next_page = False else: next_page = page_url(offset + PAGESIZE) thispage = offset / PAGESIZE maxpage = results.matches_estimated / PAGESIZE pages_to_show = set([0]) | set([thispage - 1, thispage, thispage + 1]) | set([maxpage]) if 0 == thispage: pages_to_show.remove(thispage - 1) if maxpage == thispage: pages_to_show.remove(thispage + 1) pages = [] class Page(object): def __init__(self, number, url, selected=False): self.number = number self.url = url self.selected = selected pages_in_order = list(pages_to_show) pages_in_order.sort() for page in pages_in_order: if len(pages) > 0 and page != pages[-1].number: pages.append("...") pages.append(Page(page + 1, page_url(page * PAGESIZE), page == thispage)) return { "log_lines": log_lines, "result": results, "q": q, "previous_page": previous_page, "next_page": next_page, "pages": pages, "debug": {"query": query}, }
def build_mission(self, mission): print "Building data visualisations for %s..." % mission.name for act in list(Act.Query(self.redis_conn, mission.name)): print ' ... %s' % act.title # Split the act into sections, one for each bar on the graph act_duration = act.end - act.start section_duration = act_duration // 92 # Count the number of log lines in each segment # and find the maximum number of log lines in a segment t = act.start segment_line_counts = [] max_line_count = 0 real_output_path = self.image_output_path % mission.name while t < act.end: # Load log lines for this segment query = LogLine.Query(self.redis_conn, mission.name).transcript( mission.main_transcript).range( t, t + section_duration) line_count = len(list(query)) # Store segment stats max_line_count = max(line_count, max_line_count) segment_line_counts.append( (t, t + section_duration, line_count)) t += section_duration # Make sure we have an output directory and work out where to # write the image try: os.makedirs(real_output_path) except OSError: pass graph_file = 'graph_%s_%s.png' % (mission.name, act.number) output_path = '%s/%s' % (real_output_path, graph_file) # Add initial draw command draw_commands = [ 'convert', '-size', '%dx%d' % (self.width, self.height), 'xc:transparent', '-fill', self.end_marker_colour, '-draw', "path 'M 1,1 L 10,1 L 5,8 L 1,1", '-draw', "path 'M 890,1 L 900,1 L 895,8 L 890,1", '-fill', self.graph_bar_colour, ] # Add initial image map tags image_map_id = '%s_%s_frequency_graph' % (mission.name, act.number) image_map = [ '<map id="%s" name="%s">' % (image_map_id, image_map_id) ] # Iterate over the segments and add them to the draw commands and image map for i, line in enumerate(segment_line_counts): start, end, count = line height = int( round(count / float(max(max_line_count, 1)) * self.max_bar_height)) bar_width = 6 bar_spacing = 4 top_left_x = i * (bar_width + bar_spacing) + 2 top_left_y = self.max_bar_height - height + 14 bottom_right_x = top_left_x + bar_width bottom_right_y = self.max_bar_height + 14 draw_commands.append('-draw') draw_commands.append( 'rectangle %s,%s,%s,%s' % (top_left_x, top_left_y, bottom_right_x, bottom_right_y)) if height > 0: image_map.append( '<area shape="rect" coords="%(coords)s" href="%(url)s" alt="%(alt)s">' % { "url": '/%s/%s/#show-selection' % (seconds_to_timestamp(start), seconds_to_timestamp(end)), "alt": '%d lines between %s and %s' % (count, seconds_to_timestamp(start), seconds_to_timestamp(end)), "coords": '%s,%s,%s,%s' % (top_left_x, top_left_y, bottom_right_x, bottom_right_y), }) # Output the basic graph image draw_commands.append(output_path) subprocess.call(draw_commands) # Iterate over the key scenes adding them to the graph and image map for i, key_scene in enumerate(act.key_scenes()): print ' - %s' % key_scene.title top_left_x = int( (self.graph_background_width / float(act_duration)) * (key_scene.start - act.start)) + 2 top_left_y = self.max_bar_height + 5 + 14 bottom_right_x = top_left_x + 20 bottom_right_y = top_left_y + 20 marker_image = self.key_scene_marker_files % (i + 1) subprocess.call([ 'composite', '-geometry', '+%s+%s' % (top_left_x, top_left_y), marker_image, output_path, output_path, ]) image_map.append( '<area shape="rect" coords="%(coords)s" href="%(url)s" alt="%(alt)s">' % { "url": '/%s/%s/#show-selection' % (seconds_to_timestamp(key_scene.start), seconds_to_timestamp(key_scene.end)), "alt": key_scene.title.decode('utf-8'), "coords": '%s,%s,%s,%s' % (top_left_x, top_left_y, bottom_right_x, bottom_right_y), }) # Finalise the image map image_map.append('</map>') self.redis_conn.hmset( 'act:%s:%s:stats' % (mission.name, act.number), { "image_map": "\n".join(image_map), "image_map_id": image_map_id, })
def log_line_query(self): return LogLine.Query(self.request.redis_conn, self.request.mission.name)
def get_context_data(self): # Get the query text q = self.request.GET.get('q', '') # Get the offset value try: offset = int(self.request.GET.get('offset', '0')) if offset < 0: offset = 0 except ValueError: offset = 0 # Is it a special search? special_value = self.request.redis_conn.get("special_search:%s:%s" % ( self.request.mission.name, q, )) if special_value: self.template_name = "search/special.html" return { "q": q, "text": special_value, } # Get the results from Xapian db = xappy.SearchConnection( os.path.join( settings.SITE_ROOT, '..', "xappydb", ), ) query = db.query_parse( q, default_op=db.OP_OR, deny=["mission"], ) query = db.query_filter( query, db.query_composite(db.OP_AND, [ db.query_field("mission", self.request.mission.name), db.query_field("transcript", self.request.mission.main_transcript), ])) results = db.search( query=query, startrank=offset, endrank=offset + PAGESIZE, checkatleast= -1, # everything (entire xapian db fits in memory, so this should be fine) sortby="-weight", ) # Go through the results, building a list of LogLine objects redis_conn = self.request.redis_conn log_lines = [] for result in results: transcript_name, timestamp = result.id.split(":", 1) log_line = LogLine(redis_conn, transcript_name, int(timestamp)) log_line.speaker = Character(redis_conn, transcript_name.split('/')[0], result.data['speaker_identifier'][0]) log_line.title = mark_safe( result.summarise("text", maxlen=50, ellipsis='…', strict_length=True, hl=None)) log_line.summary = mark_safe( result.summarise("text", maxlen=600, ellipsis='…', hl=('<mark>', '</mark>'))) log_lines.append(log_line) def page_url(offset): return reverse("search") + '?' + urllib.urlencode( { 'q': q.encode('utf-8'), 'offset': offset, }) if offset == 0: previous_page = False else: previous_page = page_url(offset - PAGESIZE) if offset + PAGESIZE > results.matches_estimated: next_page = False else: next_page = page_url(offset + PAGESIZE) thispage = offset / PAGESIZE maxpage = results.matches_estimated / PAGESIZE pages_to_show = set([0]) | set([thispage - 1, thispage, thispage + 1 ]) | set([maxpage]) if 0 == thispage: pages_to_show.remove(thispage - 1) if maxpage == thispage: pages_to_show.remove(thispage + 1) pages = [] class Page(object): def __init__(self, number, url, selected=False): self.number = number self.url = url self.selected = selected pages_in_order = list(pages_to_show) pages_in_order.sort() for page in pages_in_order: if len(pages) > 0 and page != pages[-1].number: pages.append('...') pages.append( Page(page + 1, page_url(page * PAGESIZE), page == thispage)) error_info = self.request.redis_conn.hgetall( "error_page:%s:%s" % ( self.request.mission.name, 'no_search_results', ), ) if not error_info: error_info = {} if error_info.has_key('classic_moment_quote'): error_quote = LogLine( self.request.redis_conn, self.request.mission.main_transcript, timestamp_to_seconds(error_info['classic_moment_quote'])) else: error_quote = None return { 'log_lines': log_lines, 'result': results, 'q': q, 'previous_page': previous_page, 'next_page': next_page, 'pages': pages, 'debug': { 'query': query, }, 'error': { 'info': error_info, 'quote': error_quote, } }
def get_context_data(self): # Get the query text q = self.request.GET.get("q", "") # Get the offset value try: offset = int(self.request.GET.get("offset", "0")) if offset < 0: offset = 0 except ValueError: offset = 0 # Is it a special search? special_value = self.request.redis_conn.get("special_search:%s:%s" % (self.request.mission.name, q)) if special_value: self.template_name = "search/special.html" return {"q": q, "text": special_value} # Get the results from Xapian db = xappy.SearchConnection(os.path.join(settings.SITE_ROOT, "..", "xappydb")) query = db.query_parse(q, default_op=db.OP_OR, deny=["mission"]) query = db.query_filter( query, db.query_composite( db.OP_AND, [ db.query_field("mission", self.request.mission.name), db.query_field("transcript", "%s/TEC" % self.request.mission.name), ], ), ) results = db.search( query=query, startrank=offset, endrank=offset + PAGESIZE, checkatleast=-1, # everything (entire xapian db fits in memory, so this should be fine) sortby="-weight", ) # Go through the results, building a list of LogLine objects redis_conn = self.request.redis_conn log_lines = [] for result in results: transcript_name, timestamp = result.id.split(":", 1) log_line = LogLine(redis_conn, transcript_name, int(timestamp)) log_line.speaker = Character( redis_conn, transcript_name.split("/")[0], result.data["speaker_identifier"][0] ) log_line.title = mark_safe( result.summarise("text", maxlen=50, ellipsis="…", strict_length=True, hl=None) ) log_line.summary = mark_safe( result.summarise("text", maxlen=600, ellipsis="…", hl=("<mark>", "</mark>")) ) log_lines.append(log_line) def page_url(offset): return reverse("search") + "?" + urllib.urlencode({"q": q, "offset": offset}) if offset == 0: previous_page = False else: previous_page = page_url(offset - PAGESIZE) if offset + PAGESIZE > results.matches_estimated: next_page = False else: next_page = page_url(offset + PAGESIZE) thispage = offset / PAGESIZE maxpage = results.matches_estimated / PAGESIZE pages_to_show = set([0]) | set([thispage - 1, thispage, thispage + 1]) | set([maxpage]) if 0 == thispage: pages_to_show.remove(thispage - 1) if maxpage == thispage: pages_to_show.remove(thispage + 1) pages = [] class Page(object): def __init__(self, number, url, selected=False): self.number = number self.url = url self.selected = selected pages_in_order = list(pages_to_show) pages_in_order.sort() for page in pages_in_order: if len(pages) > 0 and page != pages[-1].number: pages.append("...") pages.append(Page(page + 1, page_url(page * PAGESIZE), page == thispage)) error_info = self.request.redis_conn.hgetall( "error_page:%s:%s" % (self.request.mission.name, "no_search_results") ) if not error_info: error_info = {} if error_info.has_key("classic_moment_quote"): error_quote = LogLine( self.request.redis_conn, self.request.mission.main_transcript, timestamp_to_seconds(error_info["classic_moment_quote"]), ) else: error_quote = None return { "log_lines": log_lines, "result": results, "q": q, "previous_page": previous_page, "next_page": next_page, "pages": pages, "debug": {"query": query}, "error": {"info": error_info, "quote": error_quote}, }