def build_folder(tree, conn, folder, indexed_files, indexed_folders): """Build an HTML index file for a single folder.""" # Create the subfolder if it doesn't exist: ensure_folder(os.path.join(tree.target_folder, folder)) # Build the folder listing: # Name is either basename (or if that is "" name of tree) name = os.path.basename(folder) or tree.name # Generate list of folders and their mod dates: folders = [('folder', f, datetime.fromtimestamp(stat(os.path.join(tree.source_folder, folder, f)).st_mtime), # TODO: DRY with Flask route. Use url_for: _join_url(tree.name, 'source', folder, f)) for f in indexed_folders] # Generate list of files: files = [] for f in indexed_files: # Get file path on disk path = os.path.join(tree.source_folder, folder, f) file_info = stat(path) files.append((dxr.mime.icon(path), f, datetime.fromtimestamp(file_info.st_mtime), file_info.st_size, _join_url(tree.name, 'source', folder, f))) # Lay down the HTML: jinja_env = load_template_env(tree.config.temp_folder, tree.config.dxrroot) dst_path = os.path.join(tree.target_folder, folder, tree.config.directory_index) _fill_and_write_template( jinja_env, 'folder.html', dst_path, {# Common template variables: 'wwwroot': tree.config.wwwroot, 'tree': tree.name, 'tree_tuples': [(t.name, browse_url(t.name, tree.config.wwwroot, folder), t.description) for t in tree.config.sorted_tree_order], 'generated_date': tree.config.generated_date, 'paths_and_names': linked_pathname(folder, tree.name), 'filters': filter_menu_items(tree.config.filter_language), # Autofocus only at the root of each tree: 'should_autofocus_query': folder == '', # Folder template variables: 'name': name, 'path': folder, 'folders': folders, 'files': files})
def _browse_folder(tree, path, config): """Return a rendered folder listing for folder ``path``. Search for FILEs having folder == path. If any matches, render the folder listing. Otherwise, raise NotFound. """ def item_or_list(item): """If item is a list, return its first element. Otherwise, just return it. """ # TODO @pelmers: remove this function when format bumps to 20 if isinstance(item, list): return item[0] return item frozen = frozen_config(tree) files_and_folders = filtered_query( frozen['es_alias'], FILE, filter={'folder': path}, sort=[{'is_folder': 'desc'}, 'name'], size=1000000, include=['name', 'modified', 'size', 'link', 'path', 'is_binary', 'is_folder']) if not files_and_folders: raise NotFound return render_template( 'folder.html', # Common template variables: www_root=config.www_root, tree=tree, tree_tuples=[ (t['name'], url_for('.parallel', tree=t['name'], path=path), t['description']) for t in frozen_configs()], generated_date=frozen['generated_date'], google_analytics_key=config.google_analytics_key, paths_and_names=_linked_pathname(path, tree), filters=filter_menu_items( plugins_named(frozen['enabled_plugins'])), # Autofocus only at the root of each tree: should_autofocus_query=path == '', # Folder template variables: name=basename(path) or tree, path=path, files_and_folders=[ (_icon_class_name(f), f['name'], decode_es_datetime(item_or_list(f['modified'])) if 'modified' in f else None, f.get('size'), url_for('.browse', tree=tree, path=f.get('link', f['path'])[0])) for f in files_and_folders])
def _search_html(query, tree, query_text, is_case_sensitive, offset, limit, config): """Return the rendered template for search.html. """ frozen = frozen_config(tree) # Try a normal search: template_vars = { 'filters': filter_menu_items(plugins_named(frozen['enabled_plugins'])), 'generated_date': frozen['generated_date'], 'google_analytics_key': config.google_analytics_key, 'is_case_sensitive': is_case_sensitive, 'query': query_text, 'search_url': url_for('.search', tree=tree, q=query_text, redirect='false'), 'top_of_tree': url_for('.browse', tree=tree), 'tree': tree, 'tree_tuples': _tree_tuples(query_text, is_case_sensitive), 'www_root': config.www_root } return render_template('search.html', **template_vars)
def _build_common_file_template(tree, path, date, config): """Return a dictionary of the common required file template parameters. """ return { # Common template variables: 'www_root': config.www_root, 'tree': tree, 'tree_tuples': [(t['name'], url_for('.parallel', tree=t['name'], path=path), t['description']) for t in frozen_configs()], 'generated_date': date, 'google_analytics_key': config.google_analytics_key, 'filters': filter_menu_items(plugins_named( frozen_config(tree)['enabled_plugins'])), # File template variables 'paths_and_names': _linked_pathname(path, tree), 'icon_url': url_for('.static', filename='icons/mimetypes/%s.png' % icon(path)), 'path': path, 'name': basename(path) }
def _browse_folder(tree, path, config): """Return a rendered folder listing for folder ``path``. Search for FILEs having folder == path. If any matches, render the folder listing. Otherwise, raise NotFound. """ def item_or_list(item): """If item is a list, return its first element. Otherwise, just return it. """ # TODO @pelmers: remove this function when format bumps to 20 if isinstance(item, list): return item[0] return item frozen = frozen_config(tree) plugin_headers = concat_plugin_headers(plugins_named(frozen['enabled_plugins'])) files_and_folders = filtered_query( frozen['es_alias'], FILE, filter={'folder': path}, sort=[{'is_folder': 'desc'}, 'name'], size=1000000, include=['name', 'modified', 'size', 'link', 'path', 'is_binary', 'is_folder'] + plugin_headers) if not files_and_folders: raise NotFound return render_template( 'folder.html', # Common template variables: www_root=config.www_root, tree=tree, tree_tuples=_tree_tuples('.parallel', path=path), generated_date=frozen['generated_date'], google_analytics_key=config.google_analytics_key, paths_and_names=_linked_pathname(path, tree), plugin_headers=plugin_headers, filters=filter_menu_items( plugins_named(frozen['enabled_plugins'])), # Autofocus only at the root of each tree: should_autofocus_query=path == '', # Folder template variables: name=basename(path) or tree, path=path, files_and_folders=[ (_icon_class_name(f), f['name'], decode_es_datetime(item_or_list(f['modified'])) if 'modified' in f else None, f.get('size'), [f.get(h, [''])[0] for h in plugin_headers], url_for('.browse', tree=tree, path=f.get('link', f['path'])[0])) for f in files_and_folders])
def htmlify(tree, conn, icon, path, text, dst_path, plugins): """ Build HTML for path, text save it to dst_path """ # Create htmlifiers for this source htmlifiers = [] for plugin in plugins: htmlifier = plugin.htmlify(path, text) if htmlifier: htmlifiers.append(htmlifier) # Load template env = load_template_env(tree.config.temp_folder) arguments = { # Set common template variables 'wwwroot': tree.config.wwwroot, 'tree': tree.name, 'tree_tuples': [(t.name, browse_url(t.name, tree.config.wwwroot, path), t.description) for t in tree.config.sorted_tree_order], 'generated_date': tree.config.generated_date, 'filters': filter_menu_items(tree.config.filter_language), # google analytics 'google_analytics_key': tree.config.google_analytics_key, # Set file template variables 'paths_and_names': linked_pathname(path, tree.name), 'icon': icon, 'path': path, 'name': os.path.basename(path), # Someday, it would be great to stream this and not concretize the # whole thing in RAM. The template will have to quit looping through # the whole thing 3 times. 'lines': list( lines_and_annotations( build_lines(text, htmlifiers, tree.source_encoding), htmlifiers)), 'sections': build_sections(tree, conn, path, text, htmlifiers) } _fill_and_write_template(env, 'file.html', dst_path, arguments)
def _browse_folder(tree, path, config): """Return a rendered folder listing for folder ``path``. Search for FILEs having folder == path. If any matches, render the folder listing. Otherwise, raise NotFound. """ frozen = frozen_config(tree) files_and_folders = filtered_query(frozen['es_alias'], FILE, filter={'folder': path}, sort=[{ 'is_folder': 'desc' }, 'name'], size=10000, include=[ 'name', 'modified', 'size', 'link', 'path', 'is_binary', 'is_folder' ]) if not files_and_folders: raise NotFound return render_template( 'folder.html', # Common template variables: www_root=config.www_root, tree=tree, tree_tuples=[(t['name'], url_for('.parallel', tree=t['name'], path=path), t['description']) for t in frozen_configs()], generated_date=frozen['generated_date'], google_analytics_key=config.google_analytics_key, paths_and_names=_linked_pathname(path, tree), filters=filter_menu_items(plugins_named(frozen['enabled_plugins'])), # Autofocus only at the root of each tree: should_autofocus_query=path == '', # Folder template variables: name=basename(path) or tree, path=path, files_and_folders=[ (_icon_class_name(f), f['name'], decode_es_datetime(f['modified']) if 'modified' in f else None, f.get('size'), url_for('.browse', tree=tree, path=f.get('link', f['path'])[0])) for f in files_and_folders ])
def _browse_folder(tree, path, config): """Return a rendered folder listing for folder ``path``. Search for FILEs having folder == path. If any matches, render the folder listing. Otherwise, raise NotFound. """ frozen = frozen_config(tree) files_and_folders = filtered_query( frozen['es_alias'], FILE, filter={'folder': path}, sort=[{'is_folder': 'desc'}, 'name'], size=10000, exclude=['raw_data']) if not files_and_folders: raise NotFound return render_template( 'folder.html', # Common template variables: www_root=config.www_root, tree=tree, tree_tuples=[ (t['name'], url_for('.parallel', tree=t['name'], path=path), t['description']) for t in frozen_configs()], generated_date=frozen['generated_date'], google_analytics_key=config.google_analytics_key, paths_and_names=_linked_pathname(path, tree), filters=filter_menu_items( plugins_named(frozen['enabled_plugins'])), # Autofocus only at the root of each tree: should_autofocus_query=path == '', # Folder template variables: name=basename(path) or tree, path=path, files_and_folders=[ (_icon_class_name(f), f['name'], decode_es_datetime(f['modified']) if 'modified' in f else None, f.get('size'), url_for('.browse', tree=tree, path=f['path'][0]), f.get('is_binary', [False])[0]) for f in files_and_folders])
def _build_common_file_template(tree, path, is_binary, date, config): """Return a dictionary of the common required file template parameters. """ return { # Common template variables: 'www_root': config.www_root, 'tree': tree, 'tree_tuples': _tree_tuples('.parallel', path=path), 'generated_date': date, 'google_analytics_key': config.google_analytics_key, 'filters': filter_menu_items( plugins_named(frozen_config(tree)['enabled_plugins'])), # File template variables 'paths_and_names': _linked_pathname(path, tree), 'icon_url': url_for('.static', filename='icons/mimetypes/%s.png' % icon(path, is_binary)), 'path': path, 'name': basename(path) }
def htmlify(tree, conn, icon, path, text, dst_path, plugins): """ Build HTML for path, text save it to dst_path """ # Create htmlifiers for this source htmlifiers = [] for plugin in plugins: htmlifier = plugin.htmlify(path, text) if htmlifier: htmlifiers.append(htmlifier) # Load template env = load_template_env(tree.config.temp_folder) arguments = { # Set common template variables 'wwwroot': tree.config.wwwroot, 'tree': tree.name, 'tree_tuples': [(t.name, browse_url(t.name, tree.config.wwwroot, path), t.description) for t in tree.config.sorted_tree_order], 'generated_date': tree.config.generated_date, 'filters': filter_menu_items(tree.config.filter_language), # google analytics 'google_analytics_key': tree.config.google_analytics_key, # Set file template variables 'paths_and_names': linked_pathname(path, tree.name), 'icon': icon, 'path': path, 'name': os.path.basename(path), # Someday, it would be great to stream this and not concretize the # whole thing in RAM. The template will have to quit looping through # the whole thing 3 times. 'lines': list(lines_and_annotations(build_lines(text, htmlifiers, tree.source_encoding), htmlifiers)), 'sections': build_sections(tree, conn, path, text, htmlifiers) } _fill_and_write_template(env, 'file.html', dst_path, arguments)
def _build_common_file_template(tree, path, date, config): """Return a dictionary of the common required file template parameters. """ return { # Common template variables: 'www_root': config.www_root, 'tree': tree, 'tree_tuples': [(t['name'], url_for('.parallel', tree=t['name'], path=path), t['description']) for t in frozen_configs()], 'generated_date': date, 'google_analytics_key': config.google_analytics_key, 'filters': filter_menu_items( plugins_named(frozen_config(tree)['enabled_plugins'])), # File template variables 'paths_and_names': _linked_pathname(path, tree), 'icon': icon(path), 'path': path, 'name': basename(path) }
def _search_html(query, tree, query_text, offset, limit, config): """Return the rendered template for search.html. """ frozen = frozen_config(tree) # Try a normal search: template_vars = { 'filters': filter_menu_items( plugins_named(frozen['enabled_plugins'])), 'generated_date': frozen['generated_date'], 'google_analytics_key': config.google_analytics_key, 'query': query_text, 'search_url': url_for('.search', tree=tree, q=query_text, redirect='false'), 'top_of_tree': url_for('.browse', tree=tree), 'tree': tree, 'tree_tuples': _tree_tuples('.search', q=query_text), 'www_root': config.www_root} return render_template('search.html', **template_vars)
def build_folder(tree, conn, folder, indexed_files, indexed_folders): """Build an HTML index file for a single folder.""" # Create the subfolder if it doesn't exist: ensure_folder(os.path.join(tree.target_folder, folder)) # Build the folder listing: # Name is either basename (or if that is "" name of tree) name = os.path.basename(folder) or tree.name # Generate list of folders and their mod dates: folders = [ ( 'folder', f, datetime.fromtimestamp( stat(os.path.join(tree.source_folder, folder, f)).st_mtime), # TODO: DRY with Flask route. Use url_for: _join_url(tree.name, 'source', folder, f)) for f in indexed_folders ] # Generate list of files: files = [] for f in indexed_files: # Get file path on disk path = os.path.join(tree.source_folder, folder, f) file_info = stat(path) files.append( (dxr.mime.icon(path), f, datetime.fromtimestamp(file_info.st_mtime), file_info.st_size, _join_url(tree.name, 'source', folder, f))) # Lay down the HTML: jinja_env = load_template_env(tree.config.temp_folder, tree.config.dxrroot) dst_path = os.path.join(tree.target_folder, folder, tree.config.directory_index) _fill_and_write_template( jinja_env, 'folder.html', dst_path, { # Common template variables: 'wwwroot': tree.config.wwwroot, 'tree': tree.name, 'tree_tuples': [(t.name, browse_url(t.name, tree.config.wwwroot, folder), t.description) for t in tree.config.sorted_tree_order], 'generated_date': tree.config.generated_date, 'paths_and_names': linked_pathname(folder, tree.name), 'filters': filter_menu_items(tree.config.filter_language), # Autofocus only at the root of each tree: 'should_autofocus_query': folder == '', # Folder template variables: 'name': name, 'path': folder, 'folders': folders, 'files': files })
def search(tree): """Search by regex, caller, superclass, or whatever.""" # TODO: This function still does too much. querystring = request.values offset = non_negative_int(querystring.get('offset'), 0) limit = min(non_negative_int(querystring.get('limit'), 100), 1000) config = current_app.config www_root = config['WWW_ROOT'] trees = config['TREES'] # Arguments for the template: arguments = { # Common template variables 'wwwroot': www_root, 'generated_date': config['GENERATED_DATE']} error = warning = '' status_code = None if tree in trees: arguments['tree'] = tree # Connect to database conn = connect_db(tree, current_app.instance_path) if conn: # Parse the search query qtext = querystring.get('q', '') is_case_sensitive = querystring.get('case') == 'true' q = Query(conn, qtext, should_explain='explain' in querystring, is_case_sensitive=is_case_sensitive) # Try for a direct result: if querystring.get('redirect') == 'true': result = q.direct_result() if result: path, line = result # TODO: Does this escape qtext properly? return redirect( '%s/%s/source/%s?from=%s%s#%i' % (www_root, tree, path, qtext, '&case=true' if is_case_sensitive else '', line)) # Return multiple results: template = 'search.html' start = time() try: results = list(q.results(offset, limit)) except sqlite3.OperationalError as e: if e.message.startswith('REGEXP:'): # Malformed regex warning = e.message[7:] results = [] elif e.message.startswith('QUERY:'): warning = e.message[6:] results = [] else: error = 'Database error: %s' % e.message if not error: # Search template variables: arguments['time'] = time() - start arguments['query'] = qtext arguments['search_url'] = search_url(www_root, arguments['tree'], qtext, redirect=False) arguments['results'] = results arguments['offset'] = offset arguments['limit'] = limit arguments['is_case_sensitive'] = is_case_sensitive arguments['tree_tuples'] = [ (t, search_url(www_root, t, qtext, case=True if is_case_sensitive else None), description) for t, description in trees.iteritems()] else: error = 'Failed to establish database connection.' else: arguments['tree'] = trees.keys()[0] error = "Tree '%s' is not a valid tree." % tree status_code = 404 if warning or error: arguments['error'] = error or warning if querystring.get('format') == 'json': if error: # Return a non-OK code so the live search doesn't try to replace # the results with our empty ones: return jsonify(arguments), status_code or 500 # Tuples are encoded as lists in JSON, and these are not real # easy to unpack or read in Javascript. So for ease of use, we # convert to dictionaries before returning the json results. # If further discrepancies are introduced, please document them in # templating.mkd. arguments['results'] = [ {'icon': icon, 'path': path, 'lines': [{'line_number': nb, 'line': l} for nb, l in lines]} for icon, path, lines in arguments['results']] return jsonify(arguments) if error: return render_template('error.html', **arguments), status_code or 500 else: arguments['filters'] = filter_menu_items() return render_template('search.html', **arguments)
def search(tree): """Search by regex, caller, superclass, or whatever.""" # TODO: This function still does too much. querystring = request.values offset = non_negative_int(querystring.get("offset"), 0) limit = min(non_negative_int(querystring.get("limit"), 100), 1000) config = current_app.config www_root = config["WWW_ROOT"] trees = config["TREES"] google_analytics_key = config["GOOGLE_ANALYTICS_KEY"] # Arguments for the template: arguments = { # Common template variables "wwwroot": www_root, "google_analytics_key": google_analytics_key, "generated_date": config["GENERATED_DATE"], } error = warning = "" status_code = None if tree in trees: arguments["tree"] = tree # Connect to database try: conn = connect_db(join(current_app.instance_path, "trees", tree)) except sqlite3.Error: error = "Failed to establish database connection." else: # Parse the search query qtext = querystring.get("q", "") is_case_sensitive = querystring.get("case") == "true" q = Query(conn, qtext, should_explain="explain" in querystring, is_case_sensitive=is_case_sensitive) # Try for a direct result: if querystring.get("redirect") == "true": result = q.direct_result() if result: path, line = result # TODO: Does this escape qtext properly? return redirect( "%s/%s/source/%s?from=%s%s#%i" % (www_root, tree, path, qtext, "&case=true" if is_case_sensitive else "", line) ) # Return multiple results: template = "search.html" start = time() try: results = list(q.results(offset, limit)) except sqlite3.OperationalError as e: if e.message.startswith("REGEXP:"): # Malformed regex warning = e.message[7:] results = [] elif e.message.startswith("QUERY:"): warning = e.message[6:] results = [] else: error = "Database error: %s" % e.message if not error: # Search template variables: arguments["time"] = time() - start arguments["query"] = qtext arguments["search_url"] = search_url(www_root, arguments["tree"], qtext, redirect=False) arguments["results"] = results arguments["offset"] = offset arguments["limit"] = limit arguments["is_case_sensitive"] = is_case_sensitive arguments["tree_tuples"] = [ (t, search_url(www_root, t, qtext, case=True if is_case_sensitive else None), description) for t, description in trees.iteritems() ] else: arguments["tree"] = trees.keys()[0] error = "Tree '%s' is not a valid tree." % tree status_code = 404 if warning or error: arguments["error"] = error or warning if querystring.get("format") == "json": if error: # Return a non-OK code so the live search doesn't try to replace # the results with our empty ones: return jsonify(arguments), status_code or 500 # Tuples are encoded as lists in JSON, and these are not real # easy to unpack or read in Javascript. So for ease of use, we # convert to dictionaries before returning the json results. # If further discrepancies are introduced, please document them in # templating.mkd. arguments["results"] = [ {"icon": icon, "path": path, "lines": [{"line_number": nb, "line": l} for nb, l in lines]} for icon, path, lines in arguments["results"] ] return jsonify(arguments) if error: return render_template("error.html", **arguments), status_code or 500 else: arguments["filters"] = filter_menu_items(config["FILTER_LANGUAGE"]) return render_template("search.html", **arguments)
def search(tree): """Search by regex, caller, superclass, or whatever.""" # TODO: This function still does too much. querystring = request.values offset = non_negative_int(querystring.get('offset'), 0) limit = min(non_negative_int(querystring.get('limit'), 100), 1000) config = current_app.config www_root = config['WWW_ROOT'] trees = config['TREES'] # Arguments for the template: arguments = { # Common template variables 'wwwroot': www_root, 'generated_date': config['GENERATED_DATE']} error = warning = '' status_code = None if tree in trees: arguments['tree'] = tree # Connect to database try: conn = connect_db(join(current_app.instance_path, 'trees', tree)) except sqlite3.Error: error = 'Failed to establish database connection.' else: # Parse the search query qtext = querystring.get('q', '') is_case_sensitive = querystring.get('case') == 'true' q = Query(conn, qtext, should_explain='explain' in querystring, is_case_sensitive=is_case_sensitive) # Try for a direct result: if querystring.get('redirect') == 'true': result = q.direct_result() if result: path, line = result # TODO: Does this escape qtext properly? return redirect( '%s/%s/source/%s?from=%s%s#%i' % (www_root, tree, path, qtext, '&case=true' if is_case_sensitive else '', line)) # Return multiple results: template = 'search.html' start = time() try: results = list(q.results(offset, limit)) except sqlite3.OperationalError as e: if e.message.startswith('REGEXP:'): # Malformed regex warning = e.message[7:] results = [] elif e.message.startswith('QUERY:'): warning = e.message[6:] results = [] else: error = 'Database error: %s' % e.message if not error: # Search template variables: arguments['time'] = time() - start arguments['query'] = qtext arguments['search_url'] = search_url(www_root, arguments['tree'], qtext, redirect=False) arguments['results'] = results arguments['offset'] = offset arguments['limit'] = limit arguments['is_case_sensitive'] = is_case_sensitive arguments['tree_tuples'] = [ (t, search_url(www_root, t, qtext, case=True if is_case_sensitive else None), description) for t, description in trees.iteritems()] else: arguments['tree'] = trees.keys()[0] error = "Tree '%s' is not a valid tree." % tree status_code = 404 if warning or error: arguments['error'] = error or warning if querystring.get('format') == 'json': if error: # Return a non-OK code so the live search doesn't try to replace # the results with our empty ones: return jsonify(arguments), status_code or 500 # Tuples are encoded as lists in JSON, and these are not real # easy to unpack or read in Javascript. So for ease of use, we # convert to dictionaries before returning the json results. # If further discrepancies are introduced, please document them in # templating.mkd. arguments['results'] = [ {'icon': icon, 'path': path, 'lines': [{'line_number': nb, 'line': l} for nb, l in lines]} for icon, path, lines in arguments['results']] return jsonify(arguments) if error: return render_template('error.html', **arguments), status_code or 500 else: arguments['filters'] = filter_menu_items(config['FILTER_LANGUAGE']) return render_template('search.html', **arguments)