def build_folder(tree, conn, folder, indexed_files, indexed_folders):
"""Build an HTML index file for a single folder."""
# Create the subfolder if it doesn't exist:
ensure_folder(os.path.join(tree.target_folder, folder))
# Build the folder listing:
# Name is either basename (or if that is "" name of tree)
name = os.path.basename(folder) or tree.name
# Generate list of folders and their mod dates:
folders = [('folder',
f,
datetime.fromtimestamp(stat(os.path.join(tree.source_folder,
folder,
f)).st_mtime),
# TODO: DRY with Flask route. Use url_for:
_join_url(tree.name, 'source', folder, f))
for f in indexed_folders]
# Generate list of files:
files = []
for f in indexed_files:
# Get file path on disk
path = os.path.join(tree.source_folder, folder, f)
file_info = stat(path)
files.append((dxr.mime.icon(path),
f,
datetime.fromtimestamp(file_info.st_mtime),
file_info.st_size,
_join_url(tree.name, 'source', folder, f)))
# Lay down the HTML:
jinja_env = load_template_env(tree.config.temp_folder,
tree.config.dxrroot)
dst_path = os.path.join(tree.target_folder,
folder,
tree.config.directory_index)
_fill_and_write_template(
jinja_env,
'folder.html',
dst_path,
{# Common template variables:
'wwwroot': tree.config.wwwroot,
'tree': tree.name,
'tree_tuples': [(t.name,
browse_url(t.name, tree.config.wwwroot, folder),
t.description)
for t in tree.config.sorted_tree_order],
'generated_date': tree.config.generated_date,
'paths_and_names': linked_pathname(folder, tree.name),
'filters': filter_menu_items(tree.config.filter_language),
# Autofocus only at the root of each tree:
'should_autofocus_query': folder == '',
# Folder template variables:
'name': name,
'path': folder,
'folders': folders,
'files': files})
def htmlify(tree, conn, icon, path, text, dst_path, plugins):
""" Build HTML for path, text save it to dst_path """
# Create htmlifiers for this source
htmlifiers = []
for plugin in plugins:
htmlifier = plugin.htmlify(path, text)
if htmlifier:
htmlifiers.append(htmlifier)
# Load template
env = load_template_env(tree.config.temp_folder,
tree.config.template_folder)
tmpl = env.get_template('file.html')
arguments = {
# Set common template variables
'wwwroot': tree.config.wwwroot,
'tree': tree.name,
'trees': [t.name for t in tree.config.trees],
'config': tree.config.template_parameters,
'generated_date': tree.config.generated_date,
# Set file template variables
'paths_and_names': linked_pathname(path, tree.name),
'icon': icon,
'path': path,
'name': os.path.basename(path),
'lines': build_lines(tree, conn, path, text, htmlifiers),
'sections': build_sections(tree, conn, path, text, htmlifiers)
}
# Fill-in variables and dump to file with utf-8 encoding
tmpl.stream(**arguments).dump(dst_path, encoding='utf-8')
def htmlify(tree, conn, icon, path, text, dst_path, plugins):
""" Build HTML for path, text save it to dst_path """
# Create htmlifiers for this source
htmlifiers = []
for plugin in plugins:
htmlifier = plugin.htmlify(path, text)
if htmlifier:
htmlifiers.append(htmlifier)
# Load template
env = load_template_env(tree.config.temp_folder,
tree.config.template_folder)
tmpl = env.get_template('file.html')
arguments = {
# Set common template variables
'wwwroot': tree.config.wwwroot,
'tree': tree.name,
'trees': [t.name for t in tree.config.trees],
'config': tree.config.template_parameters,
'generated_date': tree.config.generated_date,
# Set file template variables
'paths_and_names': linked_pathname(path, tree.name),
'icon': icon,
'path': path,
'name': os.path.basename(path),
'lines': build_lines(tree, conn, path, text, htmlifiers),
'sections': build_sections(tree, conn, path, text, htmlifiers)
}
# Fill-in variables and dump to file with utf-8 encoding
tmpl.stream(**arguments).dump(dst_path, encoding='utf-8')
def build_folder(tree, conn, folder, indexed_files, indexed_folders):
"""Build an HTML index file for a single folder."""
# Create the subfolder if it doesn't exist:
ensure_folder(os.path.join(tree.target_folder, folder))
# Build the folder listing:
# Name is either basename (or if that is "" name of tree)
name = os.path.basename(folder) or tree.name
# Generate list of folders and their mod dates:
folders = [
(
'folder',
f,
datetime.fromtimestamp(
stat(os.path.join(tree.source_folder, folder, f)).st_mtime),
# TODO: DRY with Flask route. Use url_for:
_join_url(tree.name, 'source', folder, f)) for f in indexed_folders
]
# Generate list of files:
files = []
for f in indexed_files:
# Get file path on disk
path = os.path.join(tree.source_folder, folder, f)
file_info = stat(path)
files.append(
(dxr.mime.icon(path), f,
datetime.fromtimestamp(file_info.st_mtime), file_info.st_size,
_join_url(tree.name, 'source', folder, f)))
# Lay down the HTML:
jinja_env = load_template_env(tree.config.temp_folder,
tree.config.template_folder)
dst_path = os.path.join(tree.target_folder, folder,
tree.config.directory_index)
_fill_and_write_template(
jinja_env,
'folder.html',
dst_path,
{ # Common template variables:
'wwwroot': tree.config.wwwroot,
'tree': tree.name,
'trees': [t.name for t in tree.config.trees],
'config': tree.config.template_parameters,
'generated_date': tree.config.generated_date,
'paths_and_names': linked_pathname(folder, tree.name),
# Folder template variables:
'name': name,
'path': folder,
'folders': folders,
'files': files
})
def build_folder(tree, conn, folder, indexed_files, indexed_folders):
"""Build an HTML index file for a single folder."""
# Create the subfolder if it doesn't exist:
ensure_folder(os.path.join(tree.target_folder, folder))
# Build the folder listing:
# Name is either basename (or if that is "" name of tree)
name = os.path.basename(folder) or tree.name
# Generate list of folders and their mod dates:
folders = [('folder',
f,
datetime.fromtimestamp(stat(os.path.join(tree.source_folder,
folder,
f)).st_mtime),
# TODO: DRY with Flask route. Use url_for:
_join_url(tree.name, 'source', folder, f))
for f in indexed_folders]
# Generate list of files:
files = []
for f in indexed_files:
# Get file path on disk
path = os.path.join(tree.source_folder, folder, f)
file_info = stat(path)
files.append((dxr.mime.icon(path),
f,
datetime.fromtimestamp(file_info.st_mtime),
file_info.st_size,
_join_url(tree.name, 'source', folder, f)))
# Lay down the HTML:
jinja_env = load_template_env(tree.config.temp_folder,
tree.config.template_folder)
dst_path = os.path.join(tree.target_folder,
folder,
tree.config.directory_index)
_fill_and_write_template(
jinja_env,
'folder.html',
dst_path,
{# Common template variables:
'wwwroot': tree.config.wwwroot,
'tree': tree.name,
'trees': [t.name for t in tree.config.trees],
'config': tree.config.template_parameters,
'generated_date': tree.config.generated_date,
# Folder template variables:
'name': name,
'path': folder,
'folders': folders,
'files': files})
def htmlify(tree, conn, icon, path, text, dst_path, plugins):
""" Build HTML for path, text save it to dst_path """
# Create htmlifiers for this source
htmlifiers = []
for plugin in plugins:
htmlifier = plugin.htmlify(path, text)
if htmlifier:
htmlifiers.append(htmlifier)
# Load template
env = load_template_env(tree.config.temp_folder)
arguments = {
# Set common template variables
'wwwroot':
tree.config.wwwroot,
'tree':
tree.name,
'tree_tuples': [(t.name, browse_url(t.name, tree.config.wwwroot,
path), t.description)
for t in tree.config.sorted_tree_order],
'generated_date':
tree.config.generated_date,
'filters':
filter_menu_items(tree.config.filter_language),
# google analytics
'google_analytics_key':
tree.config.google_analytics_key,
# Set file template variables
'paths_and_names':
linked_pathname(path, tree.name),
'icon':
icon,
'path':
path,
'name':
os.path.basename(path),
# Someday, it would be great to stream this and not concretize the
# whole thing in RAM. The template will have to quit looping through
# the whole thing 3 times.
'lines':
list(
lines_and_annotations(
build_lines(text, htmlifiers, tree.source_encoding),
htmlifiers)),
'sections':
build_sections(tree, conn, path, text, htmlifiers)
}
_fill_and_write_template(env, 'file.html', dst_path, arguments)
def htmlify(tree, conn, icon, path, text, dst_path, plugins):
""" Build HTML for path, text save it to dst_path """
# Create htmlifiers for this source
htmlifiers = []
for plugin in plugins:
htmlifier = plugin.htmlify(path, text)
if htmlifier:
htmlifiers.append(htmlifier)
# Load template
env = load_template_env(tree.config.temp_folder)
arguments = {
# Set common template variables
'wwwroot': tree.config.wwwroot,
'tree': tree.name,
'tree_tuples': [(t.name,
browse_url(t.name, tree.config.wwwroot, path),
t.description)
for t in tree.config.sorted_tree_order],
'generated_date': tree.config.generated_date,
'filters': filter_menu_items(tree.config.filter_language),
# google analytics
'google_analytics_key': tree.config.google_analytics_key,
# Set file template variables
'paths_and_names': linked_pathname(path, tree.name),
'icon': icon,
'path': path,
'name': os.path.basename(path),
# Someday, it would be great to stream this and not concretize the
# whole thing in RAM. The template will have to quit looping through
# the whole thing 3 times.
'lines': list(lines_and_annotations(build_lines(text, htmlifiers,
tree.source_encoding),
htmlifiers)),
'sections': build_sections(tree, conn, path, text, htmlifiers)
}
_fill_and_write_template(env, 'file.html', dst_path, arguments)
def htmlify(tree, conn, icon, path, text, dst_path, plugins):
""" Build HTML for path, text save it to dst_path """
# Create htmlifiers for this source
htmlifiers = []
for plugin in plugins:
htmlifier = plugin.htmlify(path, text)
if htmlifier:
htmlifiers.append(htmlifier)
# Load template
env = load_template_env(tree.config.temp_folder,
tree.config.template_folder)
tmpl = env.get_template('file.html')
arguments = {
# Set common template variables
'wwwroot': tree.config.wwwroot,
'tree': tree.name,
'trees': [t.name for t in tree.config.trees],
'config': tree.config.template_parameters,
'generated_date': tree.config.generated_date,
# Set file template variables
'paths_and_names': linked_pathname(path, tree.name),
'icon': icon,
'path': path,
'name': os.path.basename(path),
# Someday, it would be great to stream this and not concretize the
# whole thing in RAM. The template will have to quit looping through
# the whole thing 3 times.
'lines': list(lines_and_annotations(build_lines(text, htmlifiers,
tree.source_encoding),
htmlifiers)),
'sections': build_sections(tree, conn, path, text, htmlifiers)
}
# Fill-in variables and dump to file with utf-8 encoding
tmpl.stream(**arguments).dump(dst_path, encoding='utf-8')
def build_instance(config_path, nb_jobs=None, tree=None, verbose=False):
"""Build a DXR instance.
:arg config_path: The path to a config file
:arg nb_jobs: The number of parallel jobs to pass into ``make``. Defaults
to whatever the config file says.
:arg tree: A single tree to build. Defaults to all the trees in the config
file.
"""
# Load configuration file
# (this will abort on inconsistencies)
overrides = {}
if nb_jobs:
# TODO: Remove this brain-dead cast when we get the types right in the
# Config object:
overrides['nb_jobs'] = str(nb_jobs)
config = Config(config_path, **overrides)
skip_indexing = 'index' in config.skip_stages
# Find trees to make, fail if requested tree isn't available
if tree:
trees = [t for t in config.trees if t.name == tree]
if len(trees) == 0:
print >> sys.stderr, "Tree '%s' is not defined in config file!" % tree
sys.exit(1)
else:
# Build everything if no tree is provided
trees = config.trees
# Create config.target_folder (if not exists)
print "Generating target folder"
ensure_folder(config.target_folder, False)
ensure_folder(config.temp_folder, not skip_indexing)
ensure_folder(config.log_folder, not skip_indexing)
jinja_env = load_template_env(config.temp_folder, config.dxrroot)
# We don't want to load config file on the server, so we just write all the
# setting into the config.py script, simple as that.
_fill_and_write_template(
jinja_env,
'config.py.jinja',
os.path.join(config.target_folder, 'config.py'),
dict(trees=repr(OrderedDict((t.name, t.description)
for t in config.trees)),
wwwroot=repr(config.wwwroot),
generated_date=repr(config.generated_date),
directory_index=repr(config.directory_index),
default_tree=repr(config.default_tree),
filter_language=repr(config.filter_language)))
# Create jinja cache folder in target folder
ensure_folder(os.path.join(config.target_folder, 'jinja_dxr_cache'))
# TODO Make open-search.xml things (or make the server so it can do them!)
# Build trees requested
ensure_folder(os.path.join(config.target_folder, 'trees'))
for tree in trees:
# Note starting time
start_time = datetime.now()
# Create folders (delete if exists)
ensure_folder(tree.target_folder, not skip_indexing) # <config.target_folder>/<tree.name>
ensure_folder(tree.object_folder, # Object folder (user defined!)
tree.source_folder != tree.object_folder) # Only clean if not the srcdir
ensure_folder(tree.temp_folder, not skip_indexing) # <config.temp_folder>/<tree.name>
# (or user defined)
ensure_folder(tree.log_folder, not skip_indexing) # <config.log_folder>/<tree.name>
# (or user defined)
# Temporary folders for plugins
ensure_folder(os.path.join(tree.temp_folder, 'plugins'), not skip_indexing)
for plugin in tree.enabled_plugins: # <tree.config>/plugins/<plugin>
ensure_folder(os.path.join(tree.temp_folder, 'plugins', plugin), not skip_indexing)
# Connect to database (exits on failure: sqlite_version, tokenizer, etc)
conn = connect_db(tree.target_folder)
if skip_indexing:
print " - Skipping indexing (due to 'index' in 'skip_stages')"
else:
# Create database tables
create_tables(tree, conn)
# Index all source files (for full text search)
# Also build all folder listing while we're at it
index_files(tree, conn)
# Build tree
build_tree(tree, conn, verbose)
# Optimize and run integrity check on database
finalize_database(conn)
# Commit database
conn.commit()
if 'html' in config.skip_stages:
print " - Skipping htmlifying (due to 'html' in 'skip_stages')"
else:
print "Building HTML for the '%s' tree." % tree.name
max_file_id = conn.execute("SELECT max(files.id) FROM files").fetchone()[0]
if config.disable_workers:
print " - Worker pool disabled (due to 'disable_workers')"
_build_html_for_file_ids(tree, 0, max_file_id)
else:
run_html_workers(tree, config, max_file_id)
# Close connection
conn.commit()
conn.close()
# Save the tree finish time
delta = datetime.now() - start_time
print "(finished building '%s' in %s)" % (tree.name, delta)
def build_instance(config_path, nb_jobs=None, tree=None):
"""Build a DXR instance.
:arg config_path: The path to a config file
:arg nb_jobs: The number of parallel jobs to pass into ``make``. Defaults
to whatever the config file says.
:arg tree: A single tree to build. Defaults to all the trees in the config
file.
"""
# Load configuration file
# (this will abort on inconsistencies)
overrides = {}
if nb_jobs:
overrides['nb_jobs'] = nb_jobs
config = dxr.utils.Config(config_path, **overrides)
# Find trees to make, fail if requested tree isn't available
if tree:
trees = [t for t in config.trees if t.name == tree]
if len(trees) == 0:
print >> sys.stderr, "Tree '%s' is not defined in config file!" % tree
sys.exit(1)
else:
# Build everything if no tree is provided
trees = config.trees
# Create config.target_folder (if not exists)
print "Generating target folder"
ensure_folder(config.target_folder, False)
ensure_folder(config.temp_folder, True)
ensure_folder(config.log_folder, True)
jinja_env = load_template_env(config.temp_folder, config.template_folder)
# We don't want to load config file on the server, so we just write all the
# setting into the config.py script, simple as that.
_fill_and_write_template(
jinja_env,
'config.py',
os.path.join(config.target_folder, 'config.py'),
dict(trees=repr([t.name for t in config.trees]),
wwwroot=repr(config.wwwroot),
template_parameters=repr(config.template_parameters),
generated_date=repr(config.generated_date),
directory_index=repr(config.directory_index)))
# Create jinja cache folder in target folder
ensure_folder(os.path.join(config.target_folder, 'jinja_dxr_cache'))
# Build root-level index.html:
ensure_folder(os.path.join(config.target_folder, 'trees'))
_fill_and_write_template(
jinja_env,
'index.html',
os.path.join(config.target_folder, 'trees', 'index.html'),
{'wwwroot': config.wwwroot,
'tree': config.trees[0].name,
'trees': [t.name for t in config.trees],
'config': config.template_parameters,
'generated_date': config.generated_date})
# TODO Make open-search.xml things (or make the server so it can do them!)
# Build trees requested
for tree in trees:
# Note starting time
started = datetime.now()
# Create folders (delete if exists)
ensure_folder(tree.target_folder, True) # <config.target_folder>/<tree.name>
ensure_folder(tree.object_folder, # Object folder (user defined!)
tree.source_folder != tree.object_folder) # Only clean if not the srcdir
ensure_folder(tree.temp_folder, True) # <config.temp_folder>/<tree.name>
# (or user defined)
ensure_folder(tree.log_folder, True) # <config.log_folder>/<tree.name>
# (or user defined)
# Temporary folders for plugins
ensure_folder(os.path.join(tree.temp_folder, 'plugins'), True)
for plugin in tree.enabled_plugins: # <tree.config>/plugins/<plugin>
ensure_folder(os.path.join(tree.temp_folder, 'plugins', plugin), True)
# Connect to database (exits on failure: sqlite_version, tokenizer, etc)
conn = dxr.utils.connect_database(tree)
# Create database tables
create_tables(tree, conn)
# Index all source files (for full text search)
# Also build all folder listing while we're at it
index_files(tree, conn)
# Build tree
build_tree(tree, conn)
# Optimize and run integrity check on database
finalize_database(conn)
# Commit database
conn.commit()
# Build html
run_html_workers(tree, conn)
# Close connection
conn.commit()
conn.close()
# Save the tree finish time
delta = datetime.now() - started
print "(finished building '%s' in %s)" % (tree.name, delta)
def build_instance(config_path, nb_jobs=None, tree=None, verbose=False):
"""Build a DXR instance.
:arg config_path: The path to a config file
:arg nb_jobs: The number of parallel jobs to pass into ``make``. Defaults
to whatever the config file says.
:arg tree: A single tree to build. Defaults to all the trees in the config
file.
"""
# Load configuration file
# (this will abort on inconsistencies)
overrides = {}
if nb_jobs:
# TODO: Remove this brain-dead cast when we get the types right in the
# Config object:
overrides['nb_jobs'] = str(nb_jobs)
config = Config(config_path, **overrides)
skip_indexing = 'index' in config.skip_stages
# Find trees to make, fail if requested tree isn't available
if tree:
trees = [t for t in config.trees if t.name == tree]
if len(trees) == 0:
print >> sys.stderr, "Tree '%s' is not defined in config file!" % tree
sys.exit(1)
else:
# Build everything if no tree is provided
trees = config.trees
# Create config.target_folder (if not exists)
print "Generating target folder"
ensure_folder(config.target_folder, False)
ensure_folder(config.temp_folder, not skip_indexing)
ensure_folder(config.log_folder, not skip_indexing)
jinja_env = load_template_env(config.temp_folder, config.dxrroot)
# We don't want to load config file on the server, so we just write all the
# setting into the config.py script, simple as that.
_fill_and_write_template(
jinja_env, 'config.py.jinja',
os.path.join(config.target_folder, 'config.py'),
dict(trees=repr(
OrderedDict((t.name, t.description) for t in config.trees)),
wwwroot=repr(config.wwwroot),
generated_date=repr(config.generated_date),
directory_index=repr(config.directory_index),
default_tree=repr(config.default_tree),
filter_language=repr(config.filter_language)))
# Create jinja cache folder in target folder
ensure_folder(os.path.join(config.target_folder, 'jinja_dxr_cache'))
# TODO Make open-search.xml things (or make the server so it can do them!)
# Build trees requested
ensure_folder(os.path.join(config.target_folder, 'trees'))
for tree in trees:
# Note starting time
start_time = datetime.now()
# Create folders (delete if exists)
ensure_folder(tree.target_folder,
not skip_indexing) # <config.target_folder>/<tree.name>
ensure_folder(
tree.object_folder, # Object folder (user defined!)
tree.source_folder !=
tree.object_folder) # Only clean if not the srcdir
ensure_folder(tree.temp_folder,
not skip_indexing) # <config.temp_folder>/<tree.name>
# (or user defined)
ensure_folder(tree.log_folder,
not skip_indexing) # <config.log_folder>/<tree.name>
# (or user defined)
# Temporary folders for plugins
ensure_folder(os.path.join(tree.temp_folder, 'plugins'),
not skip_indexing)
for plugin in tree.enabled_plugins: # <tree.config>/plugins/<plugin>
ensure_folder(os.path.join(tree.temp_folder, 'plugins', plugin),
not skip_indexing)
# Connect to database (exits on failure: sqlite_version, tokenizer, etc)
conn = connect_db(tree.target_folder)
if skip_indexing:
print " - Skipping indexing (due to 'index' in 'skip_stages')"
else:
# Create database tables
create_tables(tree, conn)
# Index all source files (for full text search)
# Also build all folder listing while we're at it
index_files(tree, conn)
# Build tree
build_tree(tree, conn, verbose)
# Optimize and run integrity check on database
finalize_database(conn)
# Commit database
conn.commit()
if 'html' in config.skip_stages:
print " - Skipping htmlifying (due to 'html' in 'skip_stages')"
else:
print "Building HTML for the '%s' tree." % tree.name
max_file_id = conn.execute(
"SELECT max(files.id) FROM files").fetchone()[0]
if config.disable_workers:
print " - Worker pool disabled (due to 'disable_workers')"
_build_html_for_file_ids(tree, 0, max_file_id)
else:
run_html_workers(tree, config, max_file_id)
# Close connection
conn.commit()
conn.close()
# Save the tree finish time
delta = datetime.now() - start_time
print "(finished building '%s' in %s)" % (tree.name, delta)
def build_instance(config_path, nb_jobs=None, tree=None):
"""Build a DXR instance.
:arg config_path: The path to a config file
:arg nb_jobs: The number of parallel jobs to pass into ``make``. Defaults
to whatever the config file says.
:arg tree: A single tree to build. Defaults to all the trees in the config
file.
"""
# Load configuration file
# (this will abort on inconsistencies)
overrides = {}
if nb_jobs:
overrides['nb_jobs'] = nb_jobs
config = Config(config_path, **overrides)
# Find trees to make, fail if requested tree isn't available
if tree:
trees = [t for t in config.trees if t.name == tree]
if len(trees) == 0:
print >> sys.stderr, "Tree '%s' is not defined in config file!" % tree
sys.exit(1)
else:
# Build everything if no tree is provided
trees = config.trees
# Create config.target_folder (if not exists)
print "Generating target folder"
ensure_folder(config.target_folder, False)
ensure_folder(config.temp_folder, True)
ensure_folder(config.log_folder, True)
jinja_env = load_template_env(config.temp_folder, config.template_folder)
# We don't want to load config file on the server, so we just write all the
# setting into the config.py script, simple as that.
_fill_and_write_template(
jinja_env,
'config.py',
os.path.join(config.target_folder, 'config.py'),
dict(trees=repr([t.name for t in config.trees]),
wwwroot=repr(config.wwwroot),
template_parameters=repr(config.template_parameters),
generated_date=repr(config.generated_date),
directory_index=repr(config.directory_index)))
# Create jinja cache folder in target folder
ensure_folder(os.path.join(config.target_folder, 'jinja_dxr_cache'))
# Build root-level index.html:
ensure_folder(os.path.join(config.target_folder, 'trees'))
_fill_and_write_template(
jinja_env,
'index.html',
os.path.join(config.target_folder, 'trees', 'index.html'),
{'wwwroot': config.wwwroot,
'tree': config.trees[0].name,
'trees': [t.name for t in config.trees],
'config': config.template_parameters,
'generated_date': config.generated_date})
# TODO Make open-search.xml things (or make the server so it can do them!)
# Build trees requested
for tree in trees:
# Note starting time
start_time = datetime.now()
# Create folders (delete if exists)
ensure_folder(tree.target_folder, True) # <config.target_folder>/<tree.name>
ensure_folder(tree.object_folder, # Object folder (user defined!)
tree.source_folder != tree.object_folder) # Only clean if not the srcdir
ensure_folder(tree.temp_folder, True) # <config.temp_folder>/<tree.name>
# (or user defined)
ensure_folder(tree.log_folder, True) # <config.log_folder>/<tree.name>
# (or user defined)
# Temporary folders for plugins
ensure_folder(os.path.join(tree.temp_folder, 'plugins'), True)
for plugin in tree.enabled_plugins: # <tree.config>/plugins/<plugin>
ensure_folder(os.path.join(tree.temp_folder, 'plugins', plugin), True)
# Connect to database (exits on failure: sqlite_version, tokenizer, etc)
conn = connect_database(tree)
# Create database tables
create_tables(tree, conn)
# Index all source files (for full text search)
# Also build all folder listing while we're at it
index_files(tree, conn)
# Build tree
build_tree(tree, conn)
# Optimize and run integrity check on database
finalize_database(conn)
# Commit database
conn.commit()
# Build html
run_html_workers(tree, conn)
# Close connection
conn.commit()
conn.close()
# Save the tree finish time
delta = datetime.now() - start_time
print "(finished building '%s' in %s)" % (tree.name, delta)
def build_folder(tree, conn, folder, indexed_files, indexed_folders):
"""Build an HTML index file for a single folder."""
# Create the subfolder if it doesn't exist:
ensure_folder(os.path.join(tree.target_folder, folder))
# Build the folder listing:
# Name is either basename (or if that is "" name of tree)
name = os.path.basename(folder) or tree.name
# Generate list of folders and their mod dates:
folders = [
(
'folder',
f,
datetime.fromtimestamp(
stat(os.path.join(tree.source_folder, folder, f)).st_mtime),
# TODO: DRY with Flask route. Use url_for:
_join_url(tree.name, 'source', folder, f)) for f in indexed_folders
]
# Generate list of files:
files = []
for f in indexed_files:
# Get file path on disk
path = os.path.join(tree.source_folder, folder, f)
file_info = stat(path)
files.append(
(dxr.mime.icon(path), f,
datetime.fromtimestamp(file_info.st_mtime), file_info.st_size,
_join_url(tree.name, 'source', folder, f)))
# Lay down the HTML:
jinja_env = load_template_env(tree.config.temp_folder)
dst_path = os.path.join(tree.target_folder, folder,
tree.config.directory_index)
_fill_and_write_template(
jinja_env,
'folder.html',
dst_path,
{ # Common template variables:
'wwwroot':
tree.config.wwwroot,
'tree':
tree.name,
'tree_tuples':
[(t.name, browse_url(t.name, tree.config.wwwroot,
folder), t.description)
for t in tree.config.sorted_tree_order],
'generated_date':
tree.config.generated_date,
'paths_and_names':
linked_pathname(folder, tree.name),
'filters':
filter_menu_items(tree.config.filter_language),
# Autofocus only at the root of each tree:
'should_autofocus_query':
folder == '',
# google analytics, if enabled
'google_analytics_key':
tree.config.google_analytics_key,
# Folder template variables:
'name':
name,
'path':
folder,
'folders':
folders,
'files':
files
})
