def test_enabled_star():
"""Make sure even plugins that aren't packages get enabled when
enabled_plugins = *.
"""
config = Config("""
[DXR]
enabled_plugins = *
[some_tree]
source_folder = /some/path
[[buglink]]
url = http://example.com/
[[python]]
python_path = /some/path
[[xpidl]]
header_path = /somewhere
""")
enabled_plugins = config.trees['some_tree'].enabled_plugins
plugin_names = [p.name for p in enabled_plugins]
ok_('urllink' in plugin_names)
eq_('core', plugin_names[0])
ok_('core' not in plugin_names[1:])
def test_unknown_options():
"""Unknown options should throw an error.
However, unknown options in a disabled plugin section should be ignored,
for consistency with our ignoring invalid values there as well.
"""
try:
config = Config("""
[DXR]
enabled_plugins = clang
disabled_plugins = buglink
smoop = 5
[mozilla-central]
source_folder = /some/path
[[buglink]]
url = http://example.com/
name = Big fat thing.
""")
except ConfigError as exc:
eq_(exc.sections, ['DXR'])
ok_('smoop' in exc.message)
else:
self.fail("Didn't raise ConfigError")
def convert(self, value, param, ctx):
# Convert to an abs path, but don't resolve symlinks, unlike parent's
# resolve_path behavior:
path = abspath(super(ConfigObject, self).convert(value, param, ctx))
with open(path, 'r') as file:
return Config(file.read(), relative_to=dirname(path))
def test_bytestring_paths():
"""Ensure source_folder and such are bytestrings, not Unicode."""
config = Config("""
[DXR]
enabled_plugins = clang
[mozilla-central]
source_folder = /some/path
""")
ok_(isinstance(config.trees['mozilla-central'].source_folder, str))
def test_deep_attrs():
"""Test traversal of multiple layers of DotDictWrappers."""
config = Config("""
[DXR]
enabled_plugins = buglink
[mozilla-central]
source_folder = /some/path
[[buglink]]
url = http://example.com/
""")
eq_(config.trees['mozilla-central'].buglink.url, 'http://example.com/')
def test_enabled_plugins():
"""Make sure enabled_plugins tolerates arbitrary whitespace between items,
maintains its order, and includes the core plugin. Make sure a plugin
section (buglink) isn't required unless its plugin is enabled.
"""
config = Config("""
[DXR]
[mozilla-central]
enabled_plugins = urllink omniglot
source_folder = /some/path
""")
eq_([p.name for p in config.trees['mozilla-central'].enabled_plugins],
['core', 'urllink', 'omniglot'])
def test_unknown_plugin():
"""Make sure we throw the right error when a plugin is unknown."""
try:
config = Config("""
[DXR]
[mozilla-central]
enabled_plugins = wonko
source_folder = /some/path
""")
except ConfigError as exc:
ok_('Never heard of plugin "wonko"' in str(exc))
else:
self.fail("An unknown plugin name passed to enabled_plugins didn't "
"raise ConfigError")
def test_and_error():
"""Make sure I'm using the ``error`` kwargs of And correctly."""
try:
config = Config("""
[DXR]
enabled_plugins = clang
workers = -5
[mozilla-central]
source_folder = /some/path
""")
except ConfigError as exc:
eq_(exc.sections, ['DXR'])
ok_('non-negative' in exc.message)
else:
self.fail("Didn't raise ConfigError")
def test_multi_word_strings():
"""Make sure strings containing whitespace aren't split up unless the spec
says they're a list.
"""
config = Config("""
[DXR]
enabled_plugins = clang buglink
[mozilla-central]
source_folder = /some/path
[[buglink]]
url = http://example.com/
name = Big fat thing.
""")
eq_(config.trees['mozilla-central'].buglink.name, 'Big fat thing.')
def test_plugin_section_required():
"""Since the buglink plugin is enabled, its required "url" arg under
[[buglink]] must be specified.
"""
try:
config = Config("""
[DXR]
enabled_plugins = buglink
[mozilla-central]
source_folder = /some/path
""")
except ConfigError as exc:
eq_(exc.sections, ['mozilla-central'])
ok_('buglink' in exc.message)
else:
self.fail("Didn't raise ConfigError")
def test_es_index():
"""Make sure that we can override es_index on a per-tree level.
"""
config = Config("""
[DXR]
es_index = test_index_{tree}
enabled_plugins =
[some_tree]
source_folder = /some/path
es_index = some_tree_index
[another_tree]
source_folder = /some/path
""")
eq_(config.es_index, 'test_index_{tree}')
eq_(config.trees['some_tree'].es_index, 'some_tree_index')
eq_(config.trees['another_tree'].es_index, 'test_index_{tree}')
def test_per_tree_workers():
"""Make sure per-tree workers are recognized and fall back to the global
default."""
config = Config("""
[DXR]
workers = 9
enabled_plugins = clang
[mozilla-central]
workers = 5
source_folder = /some/path
[flowzilla-central]
source_folder = /some/path
""")
eq_(config.trees['mozilla-central'].workers, 5)
# This should fall back to the default:
eq_(config.trees['flowzilla-central'].workers, 9)
def config(cls):
return Config(cls.config_input(cls._config_dir_path),
relative_to=cls._config_dir_path)
def build_instance(config_path, nb_jobs=None, tree=None, verbose=False):
"""Build a DXR instance.
:arg config_path: The path to a config file
:arg nb_jobs: The number of parallel jobs to pass into ``make``. Defaults
to whatever the config file says.
:arg tree: A single tree to build. Defaults to all the trees in the config
file.
"""
# Load configuration file
# (this will abort on inconsistencies)
overrides = {}
if nb_jobs:
# TODO: Remove this brain-dead cast when we get the types right in the
# Config object:
overrides['nb_jobs'] = str(nb_jobs)
config = Config(config_path, **overrides)
skip_indexing = 'index' in config.skip_stages
# Find trees to make, fail if requested tree isn't available
if tree:
trees = [t for t in config.trees if t.name == tree]
if len(trees) == 0:
print >> sys.stderr, "Tree '%s' is not defined in config file!" % tree
sys.exit(1)
else:
# Build everything if no tree is provided
trees = config.trees
# Create config.target_folder (if not exists)
print "Generating target folder"
ensure_folder(config.target_folder, False)
ensure_folder(config.temp_folder, not skip_indexing)
ensure_folder(config.log_folder, not skip_indexing)
jinja_env = load_template_env(config.temp_folder, config.dxrroot)
# We don't want to load config file on the server, so we just write all the
# setting into the config.py script, simple as that.
_fill_and_write_template(
jinja_env, 'config.py.jinja',
os.path.join(config.target_folder, 'config.py'),
dict(trees=repr(
OrderedDict((t.name, t.description) for t in config.trees)),
wwwroot=repr(config.wwwroot),
generated_date=repr(config.generated_date),
directory_index=repr(config.directory_index),
default_tree=repr(config.default_tree),
filter_language=repr(config.filter_language)))
# Create jinja cache folder in target folder
ensure_folder(os.path.join(config.target_folder, 'jinja_dxr_cache'))
# TODO Make open-search.xml things (or make the server so it can do them!)
# Build trees requested
ensure_folder(os.path.join(config.target_folder, 'trees'))
for tree in trees:
# Note starting time
start_time = datetime.now()
# Create folders (delete if exists)
ensure_folder(tree.target_folder,
not skip_indexing) # <config.target_folder>/<tree.name>
ensure_folder(
tree.object_folder, # Object folder (user defined!)
tree.source_folder !=
tree.object_folder) # Only clean if not the srcdir
ensure_folder(tree.temp_folder,
not skip_indexing) # <config.temp_folder>/<tree.name>
# (or user defined)
ensure_folder(tree.log_folder,
not skip_indexing) # <config.log_folder>/<tree.name>
# (or user defined)
# Temporary folders for plugins
ensure_folder(os.path.join(tree.temp_folder, 'plugins'),
not skip_indexing)
for plugin in tree.enabled_plugins: # <tree.config>/plugins/<plugin>
ensure_folder(os.path.join(tree.temp_folder, 'plugins', plugin),
not skip_indexing)
# Connect to database (exits on failure: sqlite_version, tokenizer, etc)
conn = connect_db(tree.target_folder)
if skip_indexing:
print " - Skipping indexing (due to 'index' in 'skip_stages')"
else:
# Create database tables
create_tables(tree, conn)
# Index all source files (for full text search)
# Also build all folder listing while we're at it
index_files(tree, conn)
# Build tree
build_tree(tree, conn, verbose)
# Optimize and run integrity check on database
finalize_database(conn)
# Commit database
conn.commit()
if 'html' in config.skip_stages:
print " - Skipping htmlifying (due to 'html' in 'skip_stages')"
else:
print "Building HTML for the '%s' tree." % tree.name
max_file_id = conn.execute(
"SELECT max(files.id) FROM files").fetchone()[0]
if config.disable_workers:
print " - Worker pool disabled (due to 'disable_workers')"
_build_html_for_file_ids(tree, 0, max_file_id)
else:
run_html_workers(tree, config, max_file_id)
# Close connection
conn.commit()
conn.close()
# Save the tree finish time
delta = datetime.now() - start_time
print "(finished building '%s' in %s)" % (tree.name, delta)
def build_instance(config_path, nb_jobs=None, tree=None):
"""Build a DXR instance.
:arg config_path: The path to a config file
:arg nb_jobs: The number of parallel jobs to pass into ``make``. Defaults
to whatever the config file says.
:arg tree: A single tree to build. Defaults to all the trees in the config
file.
"""
# Load configuration file
# (this will abort on inconsistencies)
overrides = {}
if nb_jobs:
overrides['nb_jobs'] = nb_jobs
config = Config(config_path, **overrides)
# Find trees to make, fail if requested tree isn't available
if tree:
trees = [t for t in config.trees if t.name == tree]
if len(trees) == 0:
print >> sys.stderr, "Tree '%s' is not defined in config file!" % tree
sys.exit(1)
else:
# Build everything if no tree is provided
trees = config.trees
# Create config.target_folder (if not exists)
print "Generating target folder"
ensure_folder(config.target_folder, False)
ensure_folder(config.temp_folder, True)
ensure_folder(config.log_folder, True)
jinja_env = load_template_env(config.temp_folder, config.template_folder)
# We don't want to load config file on the server, so we just write all the
# setting into the config.py script, simple as that.
_fill_and_write_template(
jinja_env,
'config.py',
os.path.join(config.target_folder, 'config.py'),
dict(trees=repr([t.name for t in config.trees]),
wwwroot=repr(config.wwwroot),
template_parameters=repr(config.template_parameters),
generated_date=repr(config.generated_date),
directory_index=repr(config.directory_index)))
# Create jinja cache folder in target folder
ensure_folder(os.path.join(config.target_folder, 'jinja_dxr_cache'))
# Build root-level index.html:
ensure_folder(os.path.join(config.target_folder, 'trees'))
_fill_and_write_template(
jinja_env,
'index.html',
os.path.join(config.target_folder, 'trees', 'index.html'),
{'wwwroot': config.wwwroot,
'tree': config.trees[0].name,
'trees': [t.name for t in config.trees],
'config': config.template_parameters,
'generated_date': config.generated_date})
# TODO Make open-search.xml things (or make the server so it can do them!)
# Build trees requested
for tree in trees:
# Note starting time
start_time = datetime.now()
# Create folders (delete if exists)
ensure_folder(tree.target_folder, True) # <config.target_folder>/<tree.name>
ensure_folder(tree.object_folder, # Object folder (user defined!)
tree.source_folder != tree.object_folder) # Only clean if not the srcdir
ensure_folder(tree.temp_folder, True) # <config.temp_folder>/<tree.name>
# (or user defined)
ensure_folder(tree.log_folder, True) # <config.log_folder>/<tree.name>
# (or user defined)
# Temporary folders for plugins
ensure_folder(os.path.join(tree.temp_folder, 'plugins'), True)
for plugin in tree.enabled_plugins: # <tree.config>/plugins/<plugin>
ensure_folder(os.path.join(tree.temp_folder, 'plugins', plugin), True)
# Connect to database (exits on failure: sqlite_version, tokenizer, etc)
conn = connect_database(tree)
# Create database tables
create_tables(tree, conn)
# Index all source files (for full text search)
# Also build all folder listing while we're at it
index_files(tree, conn)
# Build tree
build_tree(tree, conn)
# Optimize and run integrity check on database
finalize_database(conn)
# Commit database
conn.commit()
# Build html
run_html_workers(tree, conn)
# Close connection
conn.commit()
conn.close()
# Save the tree finish time
delta = datetime.now() - start_time
print "(finished building '%s' in %s)" % (tree.name, delta)