def build(self, rev):
"""Create and return the path of a new directory containing a new
deployment of the given revision of the source.
If it turns out we shouldn't deploy this build after all (perhaps
because some additional data yielded by an asynchronous build process
isn't yet available in the new format) but there hasn't been a
programming error that would warrant a more serious exception, raise
ShouldNotDeploy.
"""
VENV_NAME = 'virtualenv'
new_build_path = mkdtemp(prefix='%s-' % rev[:6],
dir=join(self.base_path, 'builds'))
try:
with cd(new_build_path):
# Make a fresh, blank virtualenv:
run('virtualenv -p {python} --no-site-packages {venv_name}',
python=self.python_path,
venv_name=VENV_NAME)
# Check out the source, and install DXR and dependencies:
run('git clone {repo} 2>/dev/null', repo=self.repo)
with cd('dxr'):
run('git checkout -q {rev}', rev=rev)
old_format = file_text('%s/dxr/dxr/format' % self._deployment_path()).rstrip()
new_format = file_text('dxr/format').rstrip()
self._format_changed_from = (old_format
if old_format != new_format
else None)
self._check_deployed_trees(old_format, new_format)
run('git submodule update -q --init --recursive')
# Make sure a malicious server didn't slip us a mickey. TODO:
# Does this recurse into submodules?
run('git fsck --no-dangling')
# Install stuff, using the new copy of peep from the checkout:
python = join(new_build_path, VENV_NAME, 'bin', 'python')
run('{python} ./peep.py install -r requirements.txt',
python=python)
# Compile nunjucks templates and cachebust static assets:
run('make static &> /dev/null')
# Quiet the complaint about there being no matches for *.so:
run('{python} setup.py install 2>/dev/null', python=python)
# After installing, you always have to re-run this, even if we
# were reusing a venv:
run('virtualenv --relocatable {venv}',
venv=join(new_build_path, VENV_NAME))
run('chmod 755 .') # mkdtemp uses a very conservative mask.
except Exception:
rmtree(new_build_path)
raise
return new_build_path
def build(self, rev):
"""Create and return the path of a new directory containing a new
deployment of the given revision of the source.
If it turns out we shouldn't deploy this build after all (perhaps
because some additional data yielded by an asynchronous build process
isn't yet available in the new format) but there hasn't been a
programming error that would warrant a more serious exception, raise
ShouldNotDeploy.
"""
VENV_NAME = 'virtualenv'
new_build_path = mkdtemp(prefix='%s-' % rev[:6],
dir=join(self.base_path, 'builds'))
try:
with cd(new_build_path):
# Make a fresh, blank virtualenv:
run('virtualenv -p {python} --no-site-packages {venv_name}',
python=self.python_path,
venv_name=VENV_NAME)
# Check out the source, and install DXR and dependencies:
run('git clone {repo} 2>/dev/null', repo=self.repo)
with cd('dxr'):
run('git checkout -q {rev}', rev=rev)
old_format = file_text('%s/dxr/dxr/format' % self._deployment_path()).rstrip()
new_format = file_text('dxr/format').rstrip()
self._format_changed_from = (old_format
if old_format != new_format
else None)
self._check_deployed_trees(old_format, new_format)
run('git submodule update -q --init --recursive')
# Make sure a malicious server didn't slip us a mickey. TODO:
# Does this recurse into submodules?
run('git fsck --no-dangling')
# Install stuff, using the new copy of peep from the checkout:
python = join(new_build_path, VENV_NAME, 'bin', 'python')
run('{python} ./peep.py install -r requirements.txt',
python=python)
# Compile nunjucks templates:
run('make templates &> /dev/null')
# Quiet the complaint about there being no matches for *.so:
run('{python} setup.py install 2>/dev/null', python=python)
# After installing, you always have to re-run this, even if we
# were reusing a venv:
run('virtualenv --relocatable {venv}',
venv=join(new_build_path, VENV_NAME))
run('chmod 755 .') # mkdtemp uses a very conservative mask.
except Exception:
rmtree(new_build_path)
raise
return new_build_path
def generate_source(cls):
code_dir = cls.code_dir()
folder = join(code_dir, 'folder')
mkdir(folder)
make_file(folder, 'file', 'some contents!')
with cd(code_dir):
check_call(['hg', 'init'])
check_call(['hg', 'add', 'folder'])
check_call(['hg', 'commit', '-m', 'Add a folder.', '-u', 'me'])
check_call(['hg', 'mv', 'folder', 'moved_folder'])
check_call(['hg', 'commit', '-m', 'Move the containing folder.', '-u', 'me'])
def rev_to_deploy(self):
"""Return the VCS revision identifier of the version we should
deploy.
If we shouldn't deploy for some reason (like if we're already at the
newest revision or nobody has pressed the Deploy button since the last
deploy), raise ShouldNotDeploy.
"""
with cd(join(self._deployment_path(), 'dxr')):
old_hash = run('git rev-parse --verify HEAD').strip()
new_hash = self.manual_rev or self._latest_successful_build()
if old_hash == new_hash:
raise ShouldNotDeploy('Version %s is already deployed.' % new_hash)
return new_hash
def rev_to_deploy(self):
"""Return the VCS revision identifier of the version we should
deploy.
If we shouldn't deploy for some reason (like if we're already at the
newest revision or nobody has pressed the Deploy button since the last
deploy), raise ShouldNotDeploy.
"""
with cd(join(self._deployment_path(), 'dxr')):
old_hash = run('git rev-parse --verify HEAD').strip()
new_hash = self.manual_rev or self._latest_successful_build()
if old_hash == new_hash:
raise ShouldNotDeploy('Version %s is already deployed.' % new_hash)
return new_hash
def install(self, new_build_path):
"""Install a build at ``self.deployment_path``, and return the path to
the build we replaced.
Avoid race conditions as much as possible. If it turns out we should
not deploy for some anticipated reason, raise ShouldNotDeploy.
"""
old_build_path = realpath(self._deployment_path())
with cd(new_build_path):
run('ln -s {points_to} {sits_at}',
points_to=new_build_path,
sits_at='new-link')
# Big, fat atomic (as in nuclear) mv:
run('mv -T new-link {dest}', dest=self._deployment_path())
# Just frobbing the symlink counts as touching the wsgi file.
return old_build_path
def install(self, new_build_path):
"""Install a build at ``self.deployment_path``, and return the path to
the build we replaced.
Avoid race conditions as much as possible. If it turns out we should
not deploy for some anticipated reason, raise ShouldNotDeploy.
"""
old_build_path = realpath(self._deployment_path())
with cd(new_build_path):
run('ln -s {points_to} {sits_at}',
points_to=new_build_path,
sits_at='new-link')
# Big, fat atomic (as in nuclear) mv:
run('mv -T new-link {dest}', dest=self._deployment_path())
# Just frobbing the symlink counts as touching the wsgi file.
return old_build_path
def dxr_index(cls):
"""Run the `dxr index` command in the config file's directory."""
with cd(cls._config_dir_path):
run('dxr index')
def teardown_class(cls):
with cd(cls._config_dir_path):
run('dxr clean')
super(DxrInstanceTestCase, cls).teardown_class()
def __init__(self, input, relative_to=None):
"""Pull in and validate a config file.
:arg input: A string or dict from which to populate the config
:arg relative_to: The dir relative to which to interpret relative paths
Raise ConfigError if the configuration is invalid.
"""
schema = Schema({
'DXR': {
Optional('temp_folder', default=abspath('dxr-temp-{tree}')):
AbsPath,
Optional('default_tree', default=None): basestring,
Optional('disabled_plugins', default=plugin_list('')): Plugins,
Optional('enabled_plugins', default=plugin_list('*')): Plugins,
Optional('generated_date',
default=datetime.utcnow()
.strftime("%a, %d %b %Y %H:%M:%S +0000")):
basestring,
Optional('log_folder', default=abspath('dxr-logs-{tree}')):
AbsPath,
Optional('workers', default=if_raises(NotImplementedError,
cpu_count,
1)):
WORKERS_VALIDATOR,
Optional('skip_stages', default=[]): WhitespaceList,
Optional('www_root', default=''): Use(lambda v: v.rstrip('/')),
Optional('google_analytics_key', default=''): basestring,
Optional('es_hosts', default='http://127.0.0.1:9200/'):
WhitespaceList,
# A semi-random name, having the tree name and format version in it.
Optional('es_index', default='dxr_{format}_{tree}_{unique}'):
basestring,
Optional('es_alias', default='dxr_{format}_{tree}'):
basestring,
Optional('es_catalog_index', default='dxr_catalog'):
basestring,
Optional('es_catalog_replicas', default=1):
Use(int, error='"es_catalog_replicas" must be an integer.'),
Optional('max_thumbnail_size', default=20000):
And(Use(int),
lambda v: v >= 0,
error='"max_thumbnail_size" must be a non-negative '
'integer.'),
Optional('es_indexing_timeout', default=60):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_timeout" must be a non-negative '
'integer.'),
Optional('es_indexing_retries', default=0):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_retries" must be a non-negative '
'integer.'),
Optional('es_refresh_interval', default=60):
Use(int, error='"es_refresh_interval" must be an integer.')
},
basestring: dict
})
# Parse the ini into nested dicts:
config_obj = ConfigObj(input.splitlines() if isinstance(input,
basestring)
else input,
list_values=False)
if not relative_to:
relative_to = getcwd()
with cd(relative_to):
try:
config = schema.validate(config_obj.dict())
except SchemaError as exc:
raise ConfigError(exc.code, ['DXR'])
self._section = config['DXR']
# Normalize enabled_plugins:
if self.enabled_plugins.is_all:
# Then explicitly enable anything that isn't explicitly
# disabled:
self._section['enabled_plugins'] = [
p for p in all_plugins_but_core().values()
if p not in self.disabled_plugins]
# Now that enabled_plugins and the other keys that TreeConfig
# depends on are filled out, make some TreeConfigs:
self.trees = OrderedDict() # name -> TreeConfig
for section in config_obj.sections:
if section != 'DXR':
try:
self.trees[section] = TreeConfig(section,
config[section],
config_obj[section].sections,
self)
except SchemaError as exc:
raise ConfigError(exc.code, [section])
# Make sure default_tree is defined:
if not self.default_tree:
self._section['default_tree'] = first(self.trees.iterkeys())
# These aren't intended for actual use; they're just to influence
# enabled_plugins of trees, and now we're done with them:
del self._section['enabled_plugins']
del self._section['disabled_plugins']
def __init__(self, input, relative_to=None):
"""Pull in and validate a config file.
:arg input: A string or dict from which to populate the config
:arg relative_to: The dir relative to which to interpret relative paths
Raise ConfigError if the configuration is invalid.
"""
schema = Schema({
'DXR': {
Optional('temp_folder', default=abspath('dxr-temp-{tree}')):
AbsPath,
Optional('default_tree', default=None):
basestring,
Optional('disabled_plugins', default=plugin_list('')):
Plugins,
Optional('enabled_plugins', default=plugin_list('*')):
Plugins,
Optional('generated_date',
default=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")):
basestring,
Optional('log_folder', default=abspath('dxr-logs-{tree}')):
AbsPath,
Optional('workers',
default=if_raises(NotImplementedError, cpu_count, 1)):
And(Use(int),
lambda v: v >= 0,
error='"workers" must be a non-negative integer.'),
Optional('skip_stages', default=[]):
WhitespaceList,
Optional('www_root', default=''):
Use(lambda v: v.rstrip('/')),
Optional('google_analytics_key', default=''):
basestring,
Optional('es_hosts', default='http://127.0.0.1:9200/'):
WhitespaceList,
# A semi-random name, having the tree name and format version in it.
Optional('es_index', default='dxr_{format}_{tree}_{unique}'):
basestring,
Optional('es_alias', default='dxr_{format}_{tree}'):
basestring,
Optional('es_catalog_index', default='dxr_catalog'):
basestring,
Optional('es_catalog_replicas', default=1):
Use(int, error='"es_catalog_replicas" must be an integer.'),
Optional('max_thumbnail_size', default=20000):
And(Use(int),
lambda v: v >= 0,
error='"max_thumbnail_size" must be a non-negative '
'integer.'),
Optional('es_indexing_timeout', default=60):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_timeout" must be a non-negative '
'integer.'),
Optional('es_indexing_retries', default=0):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_retries" must be a non-negative '
'integer.'),
Optional('es_refresh_interval', default=60):
Use(int, error='"es_refresh_interval" must be an integer.')
},
basestring: dict
})
# Parse the ini into nested dicts:
config_obj = ConfigObj(
input.splitlines() if isinstance(input, basestring) else input,
list_values=False)
if not relative_to:
relative_to = getcwd()
with cd(relative_to):
try:
config = schema.validate(config_obj.dict())
except SchemaError as exc:
raise ConfigError(exc.code, ['DXR'])
self._section = config['DXR']
# Normalize enabled_plugins:
if self.enabled_plugins.is_all:
# Then explicitly enable anything that isn't explicitly
# disabled:
self._section['enabled_plugins'] = [
p for p in all_plugins_but_core().values()
if p not in self.disabled_plugins
]
# Now that enabled_plugins and the other keys that TreeConfig
# depends on are filled out, make some TreeConfigs:
self.trees = OrderedDict() # name -> TreeConfig
for section in config_obj.sections:
if section != 'DXR':
try:
self.trees[section] = TreeConfig(
section, config[section],
config_obj[section].sections, self)
except SchemaError as exc:
raise ConfigError(exc.code, [section])
# Make sure default_tree is defined:
if not self.default_tree:
self._section['default_tree'] = first(self.trees.iterkeys())
# These aren't intended for actual use; they're just to influence
# enabled_plugins of trees, and now we're done with them:
del self._section['enabled_plugins']
del self._section['disabled_plugins']
def dxr_index(cls):
"""Run the `dxr index` command in the config file's directory."""
with cd(cls._config_dir_path):
run('dxr index')
def teardown_class(cls):
with cd(cls._config_dir_path):
run('dxr clean')
super(DxrInstanceTestCase, cls).teardown_class()
